WebClient not downloading file completely if above 50MB size

Question

I'm using WebClient for downloading files. But something fishy is going on when it tries to download larger files exactly above 50mb size, download gets completed right after 39kb and does not downloads it till the end. Anyone knows what could be the issue?

using (webClient = new WebClient())
{
    webClient.Proxy = null;
    webClient.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)");
    webClient.DownloadFileCompleted += new AsyncCompletedEventHandler(Completed);
    webClient.DownloadProgressChanged += new DownloadProgressChangedEventHandler(ProgressChanged);

    // The variable that will be holding the url address (making sure it starts with http://)
    Uri URL = urlAddress.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ? new Uri(urlAddress) : new Uri("http://" + urlAddress);

    // Start the stopwatch which we will be using to calculate the download speed
    sw.Start();

    try
    {
        // Start downloading the file
        webClient.DownloadFileAsync(URL, _mainWindow.outputDirComboBox.SelectedItem.ToString() + "\\" + _compressedClientFileDownloadRelativePath, new List<object>
        {
            _compressedClientFileDownloadRelativePath,
            _decompressedClientFileDownloadRelativePath,
            _lastDownloadedFilePath,
            _compressedSize,
            _lastCompressedFilePartRelativePath,
            _filePartsRelativePath,
            urlAddress,
            _clientFilePack,
            _clientFile
        });
    }
    catch (Exception ex)
    {
        MessageBox.Show(ex.Message);
    }
}

DownloadFileCompleted checks only for errors and when its completed but there isnt any error report going on. Only that it finishes downloading as if the download size was only 39kb

Edit1: While digging i end up here: Downloading Large Google Drive files with WebClient in C#

Looks like Google Drive gives some sort of confirm link but im not yet able to know how to putting all together working.

Edit2: I just confirmed, the 39kb are the html page confirmation

Answer 1

Thanks Everyone!

After checking this issue: Downloading Large Google Drive files with WebClient in C# I searched for getting larger files from google drive with c# and found this :

using System;
using System.IO;
using System.Net;

public static class FileDownloader
{
    private const string GOOGLE_DRIVE_DOMAIN = "drive.google.com";
    private const string GOOGLE_DRIVE_DOMAIN2 = "https://drive.google.com";

    // Normal example: FileDownloader.DownloadFileFromURLToPath( "http://example.com/file/download/link", @"C:\file.txt" );
    // Drive example: FileDownloader.DownloadFileFromURLToPath( "http://drive.google.com/file/d/FILEID/view?usp=sharing", @"C:\file.txt" );
    public static FileInfo DownloadFileFromURLToPath( string url, string path )
    {
        if( url.StartsWith( GOOGLE_DRIVE_DOMAIN ) || url.StartsWith( GOOGLE_DRIVE_DOMAIN2 ) )
            return DownloadGoogleDriveFileFromURLToPath( url, path );
        else
            return DownloadFileFromURLToPath( url, path, null );
    }

    private static FileInfo DownloadFileFromURLToPath( string url, string path, WebClient webClient )
    {
        try
        {
            if( webClient == null )
            {
                using( webClient = new WebClient() )
                {
                    webClient.DownloadFile( url, path );
                    return new FileInfo( path );
                }
            }
            else
            {
                webClient.DownloadFile( url, path );
                return new FileInfo( path );
            }
        }
        catch( WebException )
        {
            return null;
        }
    }

    // Downloading large files from Google Drive prompts a warning screen and
    // requires manual confirmation. Consider that case and try to confirm the download automatically
    // if warning prompt occurs
    private static FileInfo DownloadGoogleDriveFileFromURLToPath( string url, string path )
    {
        // You can comment the statement below if the provided url is guaranteed to be in the following format:
        // https://drive.google.com/uc?id=FILEID&export=download
        url = GetGoogleDriveDownloadLinkFromUrl( url );

        using( CookieAwareWebClient webClient = new CookieAwareWebClient() )
        {
            FileInfo downloadedFile;

            // Sometimes Drive returns an NID cookie instead of a download_warning cookie at first attempt,
            // but works in the second attempt
            for( int i = 0; i < 2; i++ )
            {
                downloadedFile = DownloadFileFromURLToPath( url, path, webClient );
                if( downloadedFile == null )
                    return null;

                // Confirmation page is around 50KB, shouldn't be larger than 60KB
                if( downloadedFile.Length > 60000 )
                    return downloadedFile;

                // Downloaded file might be the confirmation page, check it
                string content;
                using( var reader = downloadedFile.OpenText() )
                {
                    // Confirmation page starts with <!DOCTYPE html>, which can be preceeded by a newline
                    char[] header = new char[20];
                    int readCount = reader.ReadBlock( header, 0, 20 );
                    if( readCount < 20 || !( new string( header ).Contains( "<!DOCTYPE html>" ) ) )
                        return downloadedFile;

                    content = reader.ReadToEnd();
                }

                int linkIndex = content.LastIndexOf( "href=\"/uc?" );
                if( linkIndex < 0 )
                    return downloadedFile;

                linkIndex += 6;
                int linkEnd = content.IndexOf( '"', linkIndex );
                if( linkEnd < 0 )
                    return downloadedFile;

                url = "https://drive.google.com" + content.Substring( linkIndex, linkEnd - linkIndex ).Replace( "&amp;", "&" );
            }

            downloadedFile = DownloadFileFromURLToPath( url, path, webClient );

            return downloadedFile;
        }
    }

    // Handles 3 kinds of links (they can be preceeded by https://):
    // - drive.google.com/open?id=FILEID
    // - drive.google.com/file/d/FILEID/view?usp=sharing
    // - drive.google.com/uc?id=FILEID&export=download
    public static string GetGoogleDriveDownloadLinkFromUrl( string url )
    {
        int index = url.IndexOf( "id=" );
        int closingIndex;
        if( index > 0 )
        {
            index += 3;
            closingIndex = url.IndexOf( '&', index );
            if( closingIndex < 0 )
                closingIndex = url.Length;
        }
        else
        {
            index = url.IndexOf( "file/d/" );
            if( index < 0 ) // url is not in any of the supported forms
                return string.Empty;

            index += 7;

            closingIndex = url.IndexOf( '/', index );
            if( closingIndex < 0 )
            {
                closingIndex = url.IndexOf( '?', index );
                if( closingIndex < 0 )
                    closingIndex = url.Length;
            }
        }

        return string.Format( "https://drive.google.com/uc?id={0}&export=download", url.Substring( index, closingIndex - index ) );
    }
}

// Web client used for Google Drive
public class CookieAwareWebClient : WebClient
{
    private class CookieContainer
    {
        Dictionary<string, string> _cookies;

        public string this[Uri url]
        {
            get
            {
                string cookie;
                if( _cookies.TryGetValue( url.Host, out cookie ) )
                    return cookie;

                return null;
            }
            set
            {
                _cookies[url.Host] = value;
            }
        }

        public CookieContainer()
        {
            _cookies = new Dictionary<string, string>();
        }
    }

    private CookieContainer cookies;

    public CookieAwareWebClient() : base()
    {
        cookies = new CookieContainer();
    }

    protected override WebRequest GetWebRequest( Uri address )
    {
        WebRequest request = base.GetWebRequest( address );

        if( request is HttpWebRequest )
        {
            string cookie = cookies[address];
            if( cookie != null )
                ( (HttpWebRequest) request ).Headers.Set( "cookie", cookie );
        }

        return request;
    }

    protected override WebResponse GetWebResponse( WebRequest request, IAsyncResult result )
    {
        WebResponse response = base.GetWebResponse( request, result );

        string[] cookies = response.Headers.GetValues( "Set-Cookie" );
        if( cookies != null && cookies.Length > 0 )
        {
            string cookie = "";
            foreach( string c in cookies )
                cookie += c;

            this.cookies[response.ResponseUri] = cookie;
        }

        return response;
    }

    protected override WebResponse GetWebResponse( WebRequest request )
    {
        WebResponse response = base.GetWebResponse( request );

        string[] cookies = response.Headers.GetValues( "Set-Cookie" );
        if( cookies != null && cookies.Length > 0 )
        {
            string cookie = "";
            foreach( string c in cookies )
                cookie += c;

            this.cookies[response.ResponseUri] = cookie;
        }

        return response;
    }
}

With that stackoverflow question + this github code i understood it was related with both issues as google sends the confirmation page when file is bigger then 50mb. So after implementing that github code now im able to download large files with my webClient.

Answer 2

you must wait webClient.DownloadFileAsync process

else your execution fall down to the dispose command when the file is still downloading

example:

public class DownloadManager
{
    public void DownloadFile(string sourceUrl, string targetFolder)
    {
        WebClient downloader = new WebClient();
            // fake as if you are a browser making the request.
        downloader.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 8.0)");
        downloader.DownloadFileCompleted += new AsyncCompletedEventHandler(Downloader_DownloadFileCompleted);

        downloader.DownloadFileAsync(new Uri(sourceUrl), targetFolder);

    }

    private void Downloader_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e)
    {
            // display completion status.
        if (e.Error != null)
            Console.WriteLine(e.Error.Message);
        else
            Console.WriteLine("Download Completed!!!");
    }
}