0

I need some help with code. I have the following code that works really well in allowing me to programmatically log in to a vendor's website and then scrape images from it for our database. This works great. The problem is I also need to get the latest data - which is an export button on the page (exporting to excel). Using F12 and fiddler I can see that it calls a java script function. My knowledge is bit limited on the webside - I read that the client might not be the way to go for calling javascript but webbrowser? (I'm an integration specialist so SQL, SSIS, logic apps or basic coding in c# is more my thing). If I need to change to web browser how do I persist cookies from login etc. I'm coding this in azure function http trigger that will be used in logic apps flow.

public class WebClientEx : WebClient
{
    public CookieContainer CookieContainer { get; private set; }

    public WebClientEx()
    {
        CookieContainer = new CookieContainer();
    }

    protected override WebRequest GetWebRequest(Uri address)
    {
        var request = base.GetWebRequest(address);
        if (request is HttpWebRequest)
        {
            (request as HttpWebRequest).CookieContainer = CookieContainer;
        }
        return request;
    }
}

static string GetSourceForMyShowsPage(string JobId)
        {
            var sqlinsert = new SQLInsert();

            using (var client = new WebClientEx())
            {
                var values = new NameValueCollection
            {
                { "UserName", "sss" },
                { "Password", "sss" },
            };
                // Authenticate
                client.UploadValues("url/Login", values);

                //get the image url that we need to get the image in bytes - we might need to change this to a byte method?
                string responseMessage = client.DownloadString("myurl" + JobId);
                string imageurl = "";
                byte[] imageBytes;

                //search for images in string
                HtmlDocument document = new HtmlDocument();
                document.LoadHtml(responseMessage);
                document.DocumentNode.Descendants("img")
                          .Where(e =>
                          {
                              string src = e.GetAttributeValue("src", null) ?? "";
                              return !string.IsNullOrEmpty(src) && src.StartsWith(@"/Photo");
                          })
                          .ToList()
                          .ForEach(x =>
                          {
                              imageurl = x.GetAttributeValue("src", null);

                              imageurl = imageurl.Substring(0, imageurl.IndexOf("width") - 1);
                              imageurl = imageurl.Replace("amp;", "");
                              imageurl = imageurl.Replace("&amp", "");
                              imageurl = "url/" + imageurl;

                              imageBytes = client.DownloadData(imageurl);

                              string base64image = Convert.ToBase64String(imageBytes);

                              //upload data to sql table
                              sqlinsert.executeInsert(JobId, base64image);
                          });


                // Download desired page
                return client.DownloadString("url" + JobId);
            }
        }

The javascript function to be called:

function onExportClicked() {
        var data = ST.Util.serialiseForm($('#shared-tag-search-form'));
        var url = '/Reports/SharedTag';
        var form = $('<form method="POST" action="' + url + '">');
        data.push({ name: 'type', value: 'ExportToXls' });
        $.each(data, function (k, v) {
            form.append($('<input type="hidden" name="' + v.name + '" value="' + v.value + '">'));
        });
        $('body').append(form);
        form.submit();
    }

  
1
  • You're likely to have a lot more success calling the API directly. You can inspect your outgoing web traffic when loading the data to see which AJAX calls are made to load the data, then make those calls yourself in C#. Aug 9, 2022 at 2:53

1 Answer 1

0

Thanks to scatter. I am using uploadvalues of client and calling the api and then getting the data using write all bytes.

 client.BaseAddress = "url/";
            var url = "Reports/SharedTag";
            client.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36");
            client.Headers[HttpRequestHeader.ContentType] = "application/x-www-form-urlencoded";
            
            var formdata = new NameValueCollection
            {
                { "Search.Reported.StartDate", "27/07/2022" },
                { "type", "ExportToXls" },
            };

            byte[] result = client.UploadValues(url, "POST", formdata);
            File.WriteAllBytes(@"C:\Temp\Excel.xls", result);

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.