I need some help with code. I have the following code that works really well in allowing me to programmatically log in to a vendor's website and then scrape images from it for our database. This works great. The problem is I also need to get the latest data - which is an export button on the page (exporting to excel). Using F12 and fiddler I can see that it calls a java script function. My knowledge is bit limited on the webside - I read that the client might not be the way to go for calling javascript but webbrowser? (I'm an integration specialist so SQL, SSIS, logic apps or basic coding in c# is more my thing). If I need to change to web browser how do I persist cookies from login etc. I'm coding this in azure function http trigger that will be used in logic apps flow.
public class WebClientEx : WebClient
{
public CookieContainer CookieContainer { get; private set; }
public WebClientEx()
{
CookieContainer = new CookieContainer();
}
protected override WebRequest GetWebRequest(Uri address)
{
var request = base.GetWebRequest(address);
if (request is HttpWebRequest)
{
(request as HttpWebRequest).CookieContainer = CookieContainer;
}
return request;
}
}
static string GetSourceForMyShowsPage(string JobId)
{
var sqlinsert = new SQLInsert();
using (var client = new WebClientEx())
{
var values = new NameValueCollection
{
{ "UserName", "sss" },
{ "Password", "sss" },
};
// Authenticate
client.UploadValues("url/Login", values);
//get the image url that we need to get the image in bytes - we might need to change this to a byte method?
string responseMessage = client.DownloadString("myurl" + JobId);
string imageurl = "";
byte[] imageBytes;
//search for images in string
HtmlDocument document = new HtmlDocument();
document.LoadHtml(responseMessage);
document.DocumentNode.Descendants("img")
.Where(e =>
{
string src = e.GetAttributeValue("src", null) ?? "";
return !string.IsNullOrEmpty(src) && src.StartsWith(@"/Photo");
})
.ToList()
.ForEach(x =>
{
imageurl = x.GetAttributeValue("src", null);
imageurl = imageurl.Substring(0, imageurl.IndexOf("width") - 1);
imageurl = imageurl.Replace("amp;", "");
imageurl = imageurl.Replace("&", "");
imageurl = "url/" + imageurl;
imageBytes = client.DownloadData(imageurl);
string base64image = Convert.ToBase64String(imageBytes);
//upload data to sql table
sqlinsert.executeInsert(JobId, base64image);
});
// Download desired page
return client.DownloadString("url" + JobId);
}
}
The javascript function to be called:
function onExportClicked() {
var data = ST.Util.serialiseForm($('#shared-tag-search-form'));
var url = '/Reports/SharedTag';
var form = $('<form method="POST" action="' + url + '">');
data.push({ name: 'type', value: 'ExportToXls' });
$.each(data, function (k, v) {
form.append($('<input type="hidden" name="' + v.name + '" value="' + v.value + '">'));
});
$('body').append(form);
form.submit();
}
Thanks to scatter. I am using uploadvalues of client and calling the api and then getting the data using write all bytes.
client.BaseAddress = "url/";
var url = "Reports/SharedTag";
client.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36");
client.Headers[HttpRequestHeader.ContentType] = "application/x-www-form-urlencoded";
var formdata = new NameValueCollection
{
{ "Search.Reported.StartDate", "27/07/2022" },
{ "type", "ExportToXls" },
};
byte[] result = client.UploadValues(url, "POST", formdata);
File.WriteAllBytes(@"C:\Temp\Excel.xls", result);
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.