I use Html Agility Pack to parse the HTML. It's available as a NuGet package.
If you have a WebPage class defined like this:
public class WebPage
{
public string Title { get; set; }
public string PageUrl { get; set; }
public string Text { get; set; }
}
You can use the following code for parsing the HTML:
public WebPage ParseHtml(string html, Uri uri)
{
var document = new HtmlDocument();
document.LoadHtml(html);
// remove scripts
foreach (var script in document.DocumentNode.Descendants("script").ToArray())
{
script.Remove();
}
// remove styles
foreach (var style in document.DocumentNode.Descendants("style").ToArray())
{
style.Remove();
}
// remove comments
foreach (var style in document.DocumentNode.Descendants("#comment").ToArray())
{
style.Remove();
}
// sometimes </form> is not removed so we have to remove it manually
string innerText = (document.DocumentNode.InnerText ?? "").Trim().Replace("</form>", "");
var sb = new StringBuilder();
var lines = innerText.Split(new[] { Environment.NewLine, "\n" }, StringSplitOptions.RemoveEmptyEntries);
foreach (var line in lines)
{
string trimmed = StringUtils.DecodeAndRemoveSpaces(line);
if (!string.IsNullOrWhiteSpace(trimmed))
{
sb.AppendLine(trimmed);
}
}
var webPage = new WebPage { PageUrl = uri.AbsoluteUri };
var titleNode = document.DocumentNode.Descendants("title").SingleOrDefault();
if (titleNode != null)
{
webPage.Title = StringUtils.DecodeAndRemoveSpaces(titleNode.InnerText ?? "");
}
webPage.Text = sb.ToString();
return webPage;
}
And the utils class:
public class StringUtils
{
public static string DecodeAndRemoveSpaces(string text)
{
var trimed = HttpUtility.HtmlDecode(text.Trim());
trimed = trimed.Replace("\t", " ");
// replace double spaces
trimed = Regex.Replace(trimed, @"[ ]{2,}", " ");
return trimed;
}
}
Usage:
public async Task Test()
{
using (var client = new HttpClient())
{
var uri = new Uri("http://www.google.com");
string html = await client.GetStringAsync(uri);
var webPage = ParseHtml(html, uri);
}
}