C＃中处理大量图像文件的最快或最有效方法

Question

I have to read and find out the compression type of around 450,000 image files stored in our network. This is what i have got so far and it is working as desired but what i have observed is processing around 2000 files in an hr. can it be more optimized to make it more efficient.One of the reason for slowness can be since the files are read from a shared network location but there is no work around for that.

Any suggestions will be appreciated.

using System;
using System.Configuration;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.Globalization;
using System.Data;
using System.IO;
using FileHelpers;
using System.Threading.Tasks;

namespace CompressionTypeOfEDMSTiffs
{
    public static class ImageProcessor
    {
        private static readonly object lockObject = new object();

        public static void Process()
        {
            string docNumber = "";
            try
            {
                Stopwatch _stopwatch = new Stopwatch();

                _stopwatch.Start();

                var filename = ConfigurationManager.AppSettings["filePath"].Trim();

                var dtCsv = CsvEngine.CsvToDataTable(filename, ',');

                dtCsv.Columns.Add("CompressionType");
                dtCsv.Columns["CompressionType"].SetOrdinal(dtCsv.Columns.Count - 1);

                //One-by one
                //for (var rowNum = 0; rowNum < dtCsv.Rows.Count; rowNum++)
                //{
                //    var imgPath = dtCsv.Rows[rowNum]["Path"].ToString();

                //    if (!string.IsNullOrWhiteSpace(imgPath) && imgPath.LastIndexOf(".") != -1)
                //    {
                //        if (imgPath.Substring(imgPath.LastIndexOf(".")).ToUpper().Equals(".TIF"))
                //        {
                //            docNumber = dtCsv.Rows[rowNum]["DOCNUMBER"].ToString();

                //            var compression = GetCompressionTypeFromImage(imgPath);

                //            dtCsv.Rows[rowNum]["CompressionType"] = compression;

                //            //if (Enum.IsDefined(typeof(CompressionTypes), compression))
                //            //    dtCsv.Rows[rowNum]["CompressionType"] = compression.ToString();
                //            //else
                //            //{
                //            //    dtCsv.Rows[rowNum]["CompressionType"] = "UnRecognised";
                //            //}

                //            Console.WriteLine(string.Format("Counter = {0}, docnumber = {1} , path = {2},  CT = {3}", rowNum, docNumber, imgPath, compression));
                //        }
                //    }
                //}

                //Multi-tasking
                Parallel.ForEach(dtCsv.AsEnumerable(), drow =>
                {
                    var imgPath = drow["Path"].ToString();

                    if (!string.IsNullOrWhiteSpace(imgPath) && imgPath.LastIndexOf(".") != -1)
                    {
                        if (imgPath.Substring(imgPath.LastIndexOf(".")).ToUpper().Equals(".TIF"))
                        {
                            docNumber = drow["DOCNUMBER"].ToString();

                            var compression = GetCompressionTypeFromImage(imgPath);

                            drow["CompressionType"] = compression;

                            Console.WriteLine(string.Format("docnumber = {0} , path = {1},  CT = {2}", docNumber, imgPath, compression));
                        }
                    }
                });

                if (File.Exists(filename))
                    File.Delete(filename);

                //write CSV
                var tempTable = dtCsv.Copy();
                var headerRow = tempTable.NewRow();

                foreach (DataColumn col in dtCsv.Columns)
                    headerRow[col.ColumnName] = col.ColumnName;

                headerRow[headerRow.ItemArray.Length - 1] = "CompressionType";

                tempTable.Rows.InsertAt(headerRow, 0);

                CsvEngine.DataTableToCsv(tempTable, ConfigurationManager.AppSettings["filePath"].Trim());

                _stopwatch.Stop();

                Console.WriteLine(string.Format("Time elapsed in the process {0} minutes", _stopwatch.Elapsed.TotalMinutes.ToString("#.##")));

                Console.ReadLine();
            }
            catch (Exception exception)
            {
                Console.WriteLine(!string.IsNullOrWhiteSpace(docNumber)
                                      ? string.Format("Error in document No {0} and the error is {1} stack trace {2}",
                                                      docNumber, exception.Message, exception.StackTrace)
                                      : string.Format("Error is {0} stack trace {1}", exception.Message,
                                                      exception.StackTrace));
                Console.ReadLine();
            }
        }


        private static string GetCompressionTypeFromImage(string path)
        {
            string compression = "";
            try
            {
                lock (lockObject)
                {
                    using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read))
                    {
                        using (Image sourceImage = Image.FromStream(fs))
                        {
                            var compressionTagIndex = Array.IndexOf(sourceImage.PropertyIdList, 0x103);
                            PropertyItem compressionTag = sourceImage.PropertyItems[compressionTagIndex];

                            var compressionType = (CompressionTypes)Enum.Parse(typeof(CompressionTypes),
                                                            BitConverter.ToInt16(compressionTag.Value, 0).ToString(CultureInfo.InvariantCulture));

                            if (Enum.IsDefined(typeof(CompressionTypes), compressionType))
                                compression = compressionType.ToString();
                            else
                            {
                                compression = "UnRecognised";
                            }
                        }
                    } 
                }
            }
            catch (Exception exFileStream)
            {
                compression = exFileStream.Message;
            }

            return compression;
            //using (var sourceImage = Image.FromFile(path))
            //{
            //    var compressionTagIndex = Array.IndexOf(sourceImage.PropertyIdList, 0x103);
            //    PropertyItem compressionTag = sourceImage.PropertyItems[compressionTagIndex];
            //    return (CompressionTypes)Enum.Parse(typeof(CompressionTypes), BitConverter.ToInt16(compressionTag.Value, 0).ToString(CultureInfo.InvariantCulture));
            //}
        }
    }

    public enum CompressionTypes
    {
        NoCompression = 1,
        CcittGroup3 = 2,
        FacsimilecompatibleCcittGroup3 = 3,
        CcittGroup4 = 4,
        Lzw = 5,
        UnRecognised = 6,
        ExceptionInFilehandling = 7
    }
}

Answer 1

Instead of reading the files over the network, can you run your program on the server that is hosting the files?

If not, I would have one program copy files from the network to a local folder to act as a queue. Then have a 2nd program read the image from the local queue folder, determine the compression, and then delete the file. This separates the network IO time from your file processing time.

Answer 2

These are couple things that come to mind:

1. Parallel.For instead of for to go through the list.
2. async/await in .net 4.5 or .net 4 with Async CTP installed. Topic is too extensive to go over it here. You can check out Async/await here.
3. TPL DataFlow can also help parallelize the process.

Answer 3

You aren't describing a problem or a question. This is not acceptable in this community. Try editing your question and be more precise on your problem and what you want to do.

If your bottleneck is your CPU, try doing work in many threads at a time.

If your bottleneck is the file access, you can move your images in an SSD drive or in a Memory Drive and access them from there.