C# create ZipArchive with many files

Question

My method takes a List and creates a ZIP file and a log file. It works with smaller amounts of files. But when I go for a large amount of files (tested with 11,000) it throws a SystemOutOfMemory exception.

By research I learned that my method probably puts a lot of load in the memory. So I put in the part were I flush the streamwriter and the zip archive. I probably have to do something to the file stream.

What is an efficient approach to solve this problem?

Here is the code:

        public static void BackupFilesToZip(string directory, string fileFilter, string zipFilePath, bool backupInSubDir, string logFilePath, List<FileInfo> filesToBackup)
    {
        FileInfo logFile = new FileInfo(logFilePath);
        FileInfo zipFile = new FileInfo(zipFilePath);
        int numberOfFiles = filesToBackup.Count;

        if (!Directory.Exists(zipFile.DirectoryName)) Directory.CreateDirectory(zipFile.DirectoryName);

        using (FileStream zipToOpen = new FileStream(zipFile.FullName, FileMode.OpenOrCreate))
        {
            using (ZipArchive archive = new ZipArchive(zipToOpen, ZipArchiveMode.Update))
            {
                ZipArchiveEntry readmeEntry = archive.CreateEntry(logFile.Name);
                using (StreamWriter writer = new StreamWriter(readmeEntry.Open()))
                {
                    writer.WriteLine("This ZIP archive was created: " + DateTime.Now.ToString("MM/dd/yy HH:mm:ss"));
                    writer.WriteLine("ZIP File: " + zipFilePath);
                    writer.WriteLine("Source Directory: " + directory);
                    string backupInSubText = "yes";
                    if (!backupInSubDir) backupInSubText = "no";
                    writer.WriteLine("Subdirectories included: " + backupInSubText);
                    writer.WriteLine("Filter Critera: " + fileFilter);
                    writer.WriteLine("Number of Files selected: " + numberOfFiles + " (for # of files archived/skipped scroll down)");
                    writer.WriteLine("");
                    writer.WriteLine("File Log:");

                    int filesArchivedCounter = 0;
                    int filesSkippedCounter = 0;
                    int filesSum = 0;

                    TaskbarManager.Instance.SetProgressState(TaskbarProgressBarState.Normal);

                    foreach (FileInfo file in filesToBackup)
                    {
                        //ZipArchiveEntry readmeEntry = archive.CreateEntry(logFile.Name);
                        string DateTimeStampBegin = DateTime.Now.ToString("MM/dd/yy HH:mm:ss");

                        try
                        {
                            string relativePath = MakeRelativePath(directory, file.FullName);
                            archive.CreateEntryFromFile(file.FullName, relativePath);
                            writer.WriteLine(DateTimeStampBegin + " - " + DateTime.Now.ToString("HH:mm:ss") + " archived: " + file.FullName);
                            filesArchivedCounter++;
                        }
                        catch
                        {
                            writer.WriteLine(DateTimeStampBegin + " - " + " SKIPPED: " + file.FullName);
                            filesSkippedCounter++;
                        }

                        filesSum = filesSkippedCounter + filesArchivedCounter;
                        TaskbarManager.Instance.SetProgressValue(filesSum, numberOfFiles);

                        //write from memory to files every 75 items (to avoid out of memory exception)
                        if (filesSum % 75 == 0)
                        {
                            writer.Flush();
                            zipToOpen.Flush();
                        }
                    }

                    writer.WriteLine("");
                    writer.WriteLine("# of Files archived: " + filesArchivedCounter);
                    writer.WriteLine("# of Files skipped: " + filesSkippedCounter);
                }

                if (!String.IsNullOrEmpty(logFile.Name))
                {
                    if (!Directory.Exists(logFile.DirectoryName)) Directory.CreateDirectory(logFile.DirectoryName);
                    readmeEntry.ExtractToFile(logFile.FullName, true);
                }

                TaskbarManager.Instance.SetProgressState(TaskbarProgressBarState.NoProgress); 
            }
        }
    }

All the string parameters of this method are just for the log file.

Answer 1

The problem in your implementation is that you are putting the log entry at the beginning of the archive and update it after adding new files, so zip cannot be flushed. You should write log to file, not the archive, and add it to the archive when all files are added.