如何壓縮文件並通過HTTP發送而無需寫入磁盤或將完整文件加載到內存中？

Question

我的要求是將zip格式的文件上傳到某個目的地。

• 該文件需要以zip文件的形式HTTP發布到目標
• POST必須是包含其他數據的多部分消息（我認為這與我的問題無關）
• 解決方案是在.Net Core中

我有一些工作代碼; 但是，這需要我將輸入流復制到內存流中進行壓縮，從而將完整文件加載到內存中。

當文件大於可用內存時，這會成為問題，導致我的進程因OOM而終止。

這是代碼：

using System;
using System.IO;
using System.IO.Compression;
using System.Net.Http;
using System.Net.Http.Headers;

namespace HttpZipPipe
{
    public class Program
    {
        private const string sourceUrl = "https://aacapps.com/lamp/sound/amy.wav";
        private const string destUrl = "http://httpbin.org/post";    

        public static Stream CreateZipStream(Stream inputStream, string zipEntryName)
        {
            var zipStream = new MemoryStream();
            using (var zipArchive = new ZipArchive(zipStream, ZipArchiveMode.Create, true))
            {
                var zipEntry = zipArchive.CreateEntry(zipEntryName);
                using (var zipEntryStream = zipEntry.Open())
                    inputStream.CopyTo(zipEntryStream);
            }
            zipStream.Seek(0, SeekOrigin.Begin);
            return zipStream;
        }

        static void Main(string[] args)
        {
            String responseMessage;
            var Client = new HttpClient();

            using (var httpStream = Client.GetStreamAsync(sourceUrl).Result)
            using (var zipStream = CreateZipStream(httpStream, "audio.wav"))
            using (var fileContent = new StreamContent(zipStream))
            {  
                var content = new MultipartFormDataContent();
                content.Add(fileContent, "file", "file.zip");
                // ignored additional content and headers for clarity...
                responseMessage = Client.PostAsync(destUrl, content).Result.Content.ReadAsStringAsync().Result;
            }
            Console.WriteLine(responseMessage);
        }
    }
}

問題是“CreateZipStream”方法中發生了什么，特別是：

inputStream.CopyTo(zipEntryStream);

我懷疑這導致我的進程進入OOM。

有沒有辦法通過zip流將輸入流傳輸到內容而無需復制內存中的全部內容？

Answer 1

我不想使用您的服務，所以我使用常規的Web應用程序來使用Response來測試，但是這樣的事情應該可行。 你必須同時讀寫。

我還使用SharpZipLib作為nuget包。

對於文件，您必須為數據添加邊界和文件名（在SO中添加多部分消息）。 關於如何做到這一點，有一些SO帖子。

// Create the request
var request = WebRequest.Create("https://aacapps.com/lamp/sound/amy.wav");
request.UseDefaultCredentials = true;

var size = 1024 * 8;
var buffer = new byte[size];

// Create a POST request
// The four next rows I haven't tested but it's a regular POST you'll need to do
var sq = WebRequest.Create("http://httpbin.org/post");
sq.Method = "POST";
sq.ContentType = "application/octet-stream";

// Get the stream to write to
using (var res = sq.GetRequestStream())
{
    // Create zip using the httpbin request stream as writer
    using (var zip = new ZipOutputStream(res))
    {
        zip.SetLevel(3);

        // Create file entry
        var entry = new ZipEntry(ZipEntry.CleanName("amy.wav"));
        zip.PutNextEntry(entry);

        // Get the data stream
        using (var stream = request.GetResponse().GetResponseStream())
        {
            int numBytes;
            // Read the data until no more
            while ((numBytes = stream.Read(buffer, 0, size)) > 0)
            {
                // Write to the zip file buffer
                zip.Write(buffer, 0, size);
            }
        }
        zip.Close();
    }
}

// Didn't try these either but it's just reading the result if any
var resp = sq.GetResponse();
new StreamReader(resp.GetResponseStream()).ReadToEnd();