C# DownloadFileTaskAsync not downloading nested zip file

Question

Please help to a C# newbie,

I'm trying to download zipped files from HTTPS site with Script task in SSIS. Each "external" zip file contains "internal" zip file, that contains 3 txt files.

After extensive search, i've enhanced the DownloadFileTaskAsync with "await", ".Wait()" and even "while (webClient.IsBusy) but still manage to only download the empty "external" zip file.

Please help me to find a way to download full "external" file in a way it won't be empty, but will contain the "internal" zip and all 3 txt files inside of it:

    public async void Main()
    {
        WebClient webClient = new WebClient();
        webClient.Credentials = new NetworkCredential("myuser", "password", "https://example.com/Login.htm");

        webClient.DownloadFileTaskAsync(new Uri("https://example.com/#/FROMsender/EXTERNAL_20160706.zip"), @"C:\temp\Test\EXTERNAL_20160706.zip").Wait();
        while (webClient.IsBusy) Thread.Sleep(1000);

        Dts.TaskResult = (int)ScriptResults.Success;
    }

>

Answer 1

Try this one, add some headers to webClient:

public async void Main()
{
    WebClientEx webClient = new WebClientEx();  // <= use WebClientEx
    webClient.Credentials = new NetworkCredential("myuser", "password", "https://example.com/Login.htm");

    // Add headers here ------------
    webClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36");
    webClient.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
    webClient.Headers.Add("Accept-Language", "en-US,en;q=0.5");
    webClient.Headers.Add("Accept-Encoding", "gzip, deflate");
    webClient.Headers.Add("Cache-Control", "max-age=0");
    webClient.Headers.Add("DNT", "1");
    //------------------------------

    webClient.DownloadFileTaskAsync(new Uri("https://example.com/#/FROMsender/EXTERNAL_20160706.zip"), @"C:\temp\Test\EXTERNAL_20160706.zip").Wait();
    while (webClient.IsBusy) Thread.Sleep(1000);

    Dts.TaskResult = (int)ScriptResults.Success;
}

and use this class instead of WebClient:

public class WebClientEx : WebClient
{
    protected override WebRequest GetWebRequest(Uri address)
    {
      var webRequest = (HttpWebRequest) base.GetWebRequest(address);
      if (webRequest != null)
      {
        webRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
      }
      return webRequest;
    }
}