如何在c#中将大文件拆分成块？

Question

I'm making a simple file transfer sender and receiver app through the wire. What I have so far is that the sender converts the file into a byte array and sends chunks of that array to the receiver.

This works with file of up to 256mb , but this line throws a "System out of memory" exception for anything above:

byte[] buffer = StreamFile(fileName); //This is where I convert the file

I'm looking for a way to read the file in chunks then write that chunk instead of loading the whole file into a byte . How can I do this with a FileStream ?

EDIT:

Sorry, heres my crappy code so far:

    private void btnSend(object sender, EventArgs e)
    {
        Socket clientSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);


        byte[] fileName = Encoding.UTF8.GetBytes(fName); //file name
        byte[] fileData = null;
        try
        {
             fileData = StreamFile(textBox1.Text); //file
        }
        catch (OutOfMemoryException ex)
        {
            MessageBox.Show("Out of memory");
            return;
        }

        byte[] fileNameLen = BitConverter.GetBytes(fileName.Length); //length of file name
        clientData = new byte[4 + fileName.Length + fileData.Length];
        fileNameLen.CopyTo(clientData, 0);
        fileName.CopyTo(clientData, 4);
        fileData.CopyTo(clientData, 4 + fileName.Length);
        clientSock.Connect("172.16.12.91", 9050);
        clientSock.Send(clientData, 0, 4 + fileName.Length, SocketFlags.None);

        for (int i = 4 + fileName.Length; i < clientData.Length; i++)
        {
            clientSock.Send(clientData, i, 1 , SocketFlags.None);
        }

        clientSock.Close();
    }

And here's how I receive (the code was from a tutorial)

   public void ReadCallback(IAsyncResult ar)
    {

        int fileNameLen = 1;
        String content = String.Empty;
        StateObject state = (StateObject)ar.AsyncState;
        Socket handler = state.workSocket;
        int bytesRead = handler.EndReceive(ar);
        if (bytesRead > 0)
        {

            if (flag == 0)
            {
                Thread.Sleep(1000);
                fileNameLen = BitConverter.ToInt32(state.buffer, 0);
                string fileName = Encoding.UTF8.GetString(state.buffer, 4, fileNameLen);
                receivedPath = fileName;
                flag++;
            }
                if (flag >= 1)
                {
                    BinaryWriter writer = new BinaryWriter(File.Open(receivedPath, FileMode.Append));
                    if (flag == 1)
                    {
                        writer.Write(state.buffer, 4 + fileNameLen, bytesRead - (4 + fileNameLen));
                        flag++;
                    }
                    else
                        writer.Write(state.buffer, 0, bytesRead);
                        writer.Close();
                        handler.BeginReceive(state.buffer, 0, StateObject.BufferSize, 0,
                        new AsyncCallback(ReadCallback), state);
                }
        }
        else
        {
            Invoke(new MyDelegate(LabelWriter));
        }

    }

I just really want to know how I can read the file in chunks so that I dont need to convert it to a byte.

Thanks for the responses so far, I think I'm starting to get it :D

Answer 1

Just call Read repeatedly with a small buffer (I tend to use something like 16K). Note that the call to Read may end up reading a smaller amount than you request. If you're using a fixed chunk size and need the whole chunk in memory, you could just use an array of that size of course.

Without knowing how you're sending the file, it's hard to give much advice about how to structure your code, but it could be something like this:

byte[] chunk = new byte[MaxChunkSize];
while (true)
{
    int index = 0;
    // There are various different ways of structuring this bit of code.
    // Fundamentally we're trying to keep reading in to our chunk until
    // either we reach the end of the stream, or we've read everything we need.
    while (index < chunk.Length)
    {
        int bytesRead = stream.Read(chunk, index, chunk.Length - index);
        if (bytesRead == 0)
        {
            break;
        }
        index += bytesRead;
    }
    if (index != 0) // Our previous chunk may have been the last one
    {
        SendChunk(chunk, index); // index is the number of bytes in the chunk
    }
    if (index != chunk.Length) // We didn't read a full chunk: we're done
    {
        return;
    }
}

If I was more awake I'd probably find a more readable way of writing this, but it'll do for now. One option is to extract another method from the middle section:

// Attempts to read an entire chunk into the given array; returns the size of
// chunk actually read.
int ReadChunk(Stream stream, byte[] chunk)
{
    int index = 0;
    while (index < chunk.Length)
    {
        int bytesRead = stream.Read(chunk, index, chunk.Length - index);
        if (bytesRead == 0)
        {
            break;
        }
        index += bytesRead;
    }
    return index;
}

Answer 2

var b = new byte[1<<15]; // 32k
while((count = inStream.Read(b, 0, b.Length)) > 0)
{
  outStream.Write(b, 0, count);
}

Answer 3

 public static IEnumerable<byte[]> SplitStreamIntoChunks(Stream stream, int chunkSize)
    {
      var bytesRemaining = stream.Length;
      while (bytesRemaining > 0)
      {
        var size = Math.Min((int) bytesRemaining, chunkSize);
        var buffer = new byte[size];
        var bytesRead = stream.Read(buffer, 0, size);
        if (bytesRead <= 0)
          break;
        yield return buffer;
        bytesRemaining -= bytesRead;
      }
    }