C＃缓冲的zip流代理

Question

I need to send very large zip files via http. The files are too large to write to the file system or memory.I would like to zip on the fly and send back via http output stream. I'm using a custom web server that will only take a stream as a response, I don't have access to the http response output stream. (response.Send(stream)).

My idea is to create a helper/proxy stream that can be passed into the response.Send method. (See ZipCreateStream below) Basically this encapsulates a memory stream, the files to be zipped and the archive. The response.Send method will buffer copy the stream passed in calling stream.Read, the ZipCreateStream implements a Read method that will zip a buffered chunk into it's memory stream (the archive) then copy that to the buffer passed in by response.Send.

This works great until I try to truncate the memory stream in ZipCreateStream Read method. What I'm trying to do there is keep the memory usage low, after the bytes in the temporary stream are copied I want to "clear" the memory stream. When I do this the zip file ends up corrupted.

Any help/thoughts would be much appreciated! I'm truly stumped on this one.

Note: I'm using SharpZipLib for zipping.

public class ZipCreateStream : Stream
{
    readonly string[] entries;
    readonly string[] files;
    ZipOutputStream archive;
    ZipEntry archiveEntry;
    int fileNumber;
    Stream fileStream;
    volatile MemoryStream stream;
    bool hasDisposed;
    int streamOffset;
    readonly bool truncate;

    public ZipCreateStream(string[] files, string[] entries, int compressionLevel = 0, bool truncate = false)
    {
        this.files = files;
        this.entries = entries;
        if (files.Length != entries.Length)
        {
            throw new ArgumentException("Files and entries mismatch");
        }
        this.truncate = truncate;
        stream = new MemoryStream();
        archive = new ZipOutputStream(stream);
        archive.SetLevel(compressionLevel);
    }

    public override bool CanRead
    {
        get
        {
            return true;
        }
    }

    /// <summary>
    ///     When overridden in a derived class, gets a value indicating whether the current stream supports seeking.
    /// </summary>
    /// <returns>
    ///     true if the stream supports seeking; otherwise, false.
    /// </returns>
    public override bool CanSeek
    {
        get
        {
            return false;
        }
    }

    /// <summary>
    ///     When overridden in a derived class, gets a value indicating whether the current stream supports writing.
    /// </summary>
    /// <returns>
    ///     true if the stream supports writing; otherwise, false.
    /// </returns>
    public override bool CanWrite
    {
        get
        {
            return false;
        }
    }

    /// <summary>
    ///     When overridden in a derived class, gets the length in bytes of the stream.
    /// </summary>
    /// <returns>
    ///     A long value representing the length of the stream in bytes.
    /// </returns>
    /// <exception cref="T:System.NotSupportedException">A class derived from Stream does not support seeking. </exception>
    /// <exception cref="T:System.ObjectDisposedException">Methods were called after the stream was closed. </exception>
    public override long Length
    {
        get
        {
            throw new NotSupportedException();
        }
    }

    /// <summary>
    ///     When overridden in a derived class, gets or sets the position within the current stream.
    /// </summary>
    /// <returns>
    ///     The current position within the stream.
    /// </returns>
    /// <exception cref="T:System.IO.IOException">An I/O error occurs. </exception>
    /// <exception cref="T:System.NotSupportedException">The stream does not support seeking. </exception>
    /// <exception cref="T:System.ObjectDisposedException">Methods were called after the stream was closed. </exception>
    public override long Position { get; set; }

    protected override void Dispose(bool disposing)
    {
        archive.Dispose();
        stream.Dispose();
        if (fileStream != null)
        {
            fileStream.Dispose();
        }
        base.Dispose(disposing);
    }

    /// <summary>
    ///     When overridden in a derived class, clears all buffers for this stream and causes any buffered data to be written
    ///     to the underlying device.
    /// </summary>
    /// <exception cref="T:System.IO.IOException">An I/O error occurs. </exception>
    public override void Flush()
    {
        if (stream == null)
        {
            return;
        }
        stream.Flush();
    }

    /// <summary>
    ///     When overridden in a derived class, sets the position within the current stream.
    /// </summary>
    /// <returns>
    ///     The new position within the current stream.
    /// </returns>
    /// <param name="offset">A byte offset relative to the <paramref name="origin" /> parameter. </param>
    /// <param name="origin">
    ///     A value of type <see cref="T:System.IO.SeekOrigin" /> indicating the reference point used to
    ///     obtain the new position.
    /// </param>
    /// <exception cref="T:System.IO.IOException">An I/O error occurs. </exception>
    /// <exception cref="T:System.NotSupportedException">
    ///     The stream does not support seeking, such as if the stream is
    ///     constructed from a pipe or console output.
    /// </exception>
    /// <exception cref="T:System.ObjectDisposedException">Methods were called after the stream was closed. </exception>
    public override long Seek(long offset, SeekOrigin origin)
    {
        return stream.Seek(offset, origin);
    }

    /// <summary>
    ///     When overridden in a derived class, sets the length of the current stream.
    /// </summary>
    /// <param name="value">The desired length of the current stream in bytes. </param>
    /// <exception cref="T:System.IO.IOException">An I/O error occurs. </exception>
    /// <exception cref="T:System.NotSupportedException">
    ///     The stream does not support both writing and seeking, such as if the
    ///     stream is constructed from a pipe or console output.
    /// </exception>
    /// <exception cref="T:System.ObjectDisposedException">Methods were called after the stream was closed. </exception>
    public override void SetLength(long value)
    {
        stream.SetLength(value);
    }

    /// <summary>
    ///     When overridden in a derived class, reads a sequence of bytes from the current stream and advances the position
    ///     within the stream by the number of bytes read.
    /// </summary>
    /// <returns>
    ///     The total number of bytes read into the buffer. This can be less than the number of bytes requested if that many
    ///     bytes are not currently available, or zero (0) if the end of the stream has been reached.
    /// </returns>
    /// <param name="buffer">
    ///     An array of bytes. When this method returns, the buffer contains the specified byte array with the
    ///     values between <paramref name="offset" /> and (<paramref name="offset" /> + <paramref name="count" /> - 1) replaced
    ///     by the bytes read from the current source.
    /// </param>
    /// <param name="offset">
    ///     The zero-based byte offset in <paramref name="buffer" /> at which to begin storing the data read
    ///     from the current stream.
    /// </param>
    /// <param name="count">The maximum number of bytes to be read from the current stream. </param>
    /// <exception cref="T:System.ArgumentException">
    ///     The sum of <paramref name="offset" /> and <paramref name="count" /> is
    ///     larger than the buffer length.
    /// </exception>
    /// <exception cref="T:System.ArgumentNullException"><paramref name="buffer" /> is null. </exception>
    /// <exception cref="T:System.ArgumentOutOfRangeException">
    ///     <paramref name="offset" /> or <paramref name="count" /> is
    ///     negative.
    /// </exception>
    /// <exception cref="T:System.IO.IOException">An I/O error occurs. </exception>
    /// <exception cref="T:System.NotSupportedException">The stream does not support reading. </exception>
    /// <exception cref="T:System.ObjectDisposedException">Methods were called after the stream was closed. </exception>
    public override int Read(byte[] buffer, int offset, int count)
    {
        // Get the next set of buffered data from internal buffer
        if (offset != 0)
        {
            throw new ArgumentException("Not seekable");
        }

        if (streamOffset == stream.Length)
        {
            // when all buffered data is copied, clear the memory stream
            if (truncate)
            {
                streamOffset = 0;
                stream.SetLength(0);

            }
            if (fileStream != null && fileStream.Position == fileStream.Length)
            {
                fileNumber++;
                fileStream.Dispose();
                fileStream = null;
            }
            if (fileNumber < files.Length)
            {
                if (fileStream == null)
                {
                    string file = files[fileNumber];
                    string entry = entries[fileNumber];
                    fileStream = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read);
                    archiveEntry = new ZipEntry(ZipEntry.CleanName(entry));
                    archive.PutNextEntry(archiveEntry);
                }

                byte[] writebuffer = new byte[buffer.Length];

                while (stream.Length - streamOffset < buffer.Length)
                {
                    int bytesRead = fileStream.Read(writebuffer, 0, writebuffer.Length);
                    if (bytesRead > 0)
                    {
                        archive.Write(writebuffer, 0, bytesRead);
                    }
                    else
                    {
                        archive.Flush();
                        break;
                    }
                }
            }
        }
        if (streamOffset == stream.Length && !hasDisposed)
        {
            hasDisposed = true;
            archive.Finish();
        }

        stream.Seek(streamOffset, SeekOrigin.Begin);
        int readCount = stream.Read(buffer, 0, count);
        streamOffset += readCount;

        return readCount;
    }

    /// <summary>
    ///     When overridden in a derived class, writes a sequence of bytes to the current stream and advances the current
    ///     position within this stream by the number of bytes written.
    /// </summary>
    /// <param name="buffer">
    ///     An array of bytes. This method copies <paramref name="count" /> bytes from
    ///     <paramref name="buffer" /> to the current stream.
    /// </param>
    /// <param name="offset">
    ///     The zero-based byte offset in <paramref name="buffer" /> at which to begin copying bytes to the
    ///     current stream.
    /// </param>
    /// <param name="count">The number of bytes to be written to the current stream. </param>
    public override void Write(byte[] buffer, int offset, int count)
    {
        throw new NotSupportedException();
    }
}

Answer 1

The problem is in Position property. Below you can find 2 working classes for this task. They use standard .NET ZipArchive class:

public class ThroughZipStream : Stream
{
    private readonly ThroughMemoryStream _ms = new ThroughMemoryStream();

    private readonly ZipArchive _zip;
    private readonly Stream _entryStream;
    private readonly Stream _input;
    private byte[] _buffer = new byte[10000]; // buffer to read from input stream

    public ThroughZipStream(Stream input, string entryName)
    {
        _input = input;

        _zip = new ZipArchive(_ms, ZipArchiveMode.Create, true);
        var entry = _zip.CreateEntry(entryName);
        _entryStream = entry.Open();
    }

    public override int Read(byte[] buffer, int offset, int count)
    {
        while (_ms.Length < count && _buffer != null)
        {
            var len = _input.Read(_buffer, 0, _buffer.Length);
            if (len == 0)
            {
                _entryStream.Dispose();
                _zip.Dispose();
                _buffer = null;
                break;
            }
            _entryStream.Write(_buffer, 0, len);
        }

        return _ms.Read(buffer, offset, count);
    }

    public override bool CanRead { get { return true; } }
    public override bool CanSeek { get { return false; } } // we can't seek
    public override bool CanWrite { get { return false; } }

    public override void Flush() { }
    public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } }
    public override long Length { get { throw new NotImplementedException(); } }
    public override long Seek(long offset, SeekOrigin origin) { throw new NotImplementedException(); }
    public override void SetLength(long value) { throw new NotImplementedException(); }
    public override void Write(byte[] buffer, int offset, int count) { throw new NotImplementedException(); }
}

public class ThroughMemoryStream : MemoryStream
{
    private long _position;

    public override bool CanSeek { get { return false; } }

    public override void Write(byte[] buffer, int offset, int count)
    {
        base.Write(buffer, offset, count);
        _position += count;
    }

    public override int Read(byte[] buffer, int offset, int count)
    {
        var msBuffer = GetBuffer();
        var realCount = Math.Min(Length, count);
        Array.Copy(msBuffer, 0, buffer, 0, realCount);

        Array.Copy(msBuffer, realCount, msBuffer, 0, Length - realCount);
        SetLength(Length - realCount);

        return (int)realCount;
    }

    public override long Position
    {
        get { return _position; } 
        set { throw new NotImplementedException(); }
    }
}

Usage:

var compressedStream = new ThroughZipStream(fileStream, "somefile.txt")

PS: I had the same problem and couldn't google any solution. It was a big surprise for me, I think this way is the best one to pipeline a stream.