I download a html site with WebClient and downloadString() and then I try to get all link between the and them into a list.
After a few tries and 2 hours of work, 1 time I got all links, sometimes I just get one and sometimes I get none.
Here's a Sample of my code - I just let the Catch Block away for better readability.
List<string> getLinks = new List<string>();
for (int i = 0; i < wikiUrls.Length; i++)
{
try
{
string download = client.DownloadString(wikiUrls[i]);
string searchForDiv = "<div class=\"wiki\">";
int firstCharacter = download.IndexOf(searchForDiv);
//if wiki doens't exists, go to next element of for loop
if (firstCharacter == -1)
continue;
else
{
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(download);
string nodes = String.Empty;
var div = document.DocumentNode.SelectSingleNode("//div[@class=\"wiki\"]");
if (div != null)
{
getLinks = div.Descendants("a").Select(node => node.GetAttributeValue("href", "Not found \n")).ToList();
output.Text = string.Join(" ", getLinks);
}
}
}