LINQ查询的性能问题

Question

我有一个从目录中获取文件列表的功能，然后从列表中搜索文件名以查找匹配项。 表演糟透了。

这是函数：

public List<fileStatus> checkFilesStatus(List<string> permitNumbers, string serverDirectory, fileType type)
    {
        XmlConfigurator.Configure();
        log.Debug(string.Format("Beginning checkFilesStatus with following parameters > permitNumbers: {0} > serverDirectory: {1} > type: {2}", string.Join(",", permitNumbers.ToArray()), serverDirectory, type.ToString()));
        List<fileStatus> results = new List<fileStatus>();
        DirectoryInfo dirInfo = new DirectoryInfo(serverDirectory);
        if (dirInfo.Exists)
        {
            // GET LIST OF ALL FILES IN DIRECTORY
            string[] files = System.IO.Directory.GetFiles(serverDirectory, "*", System.IO.SearchOption.AllDirectories);

            log.Debug(string.Format("List of all files in directory: {0}", string.Join(",", files)));


            if (files.Length > 0 && permitNumbers.Count > 0)
            {
                log.Debug("Checking for matching files");
                // CHECK FOR MATCHING FILES
                switch (type)
                {
                    case fileType.Well:

                        var matchingFiles = (from f in files
                                             where f.Substring(f.LastIndexOf("\\") + 1).Length > 4
                                             where permitNumbers.Contains(f.Substring(f.LastIndexOf("\\") + 1, 5))
                                             select new fileStatus(fileType.Well, f.Substring(f.LastIndexOf("\\") + 1, 5), 1, f.Substring(f.LastIndexOf("\\") + 1)));


                        var permitNumbersWithMatches = (from x in matchingFiles
                                                       select x.PermitNumber);

                        var nonMatchingFiles = (from p in permitNumbers
                                                where !permitNumbersWithMatches.Contains(p)
                                                select new fileStatus(fileType.Well, p, 0, string.Empty));

                        results.AddRange(matchingFiles);
                        results.AddRange(nonMatchingFiles);

                        break;
                    case fileType.DrillerLog:
                        matchingFiles = (from f in files
                                         where f.Substring(f.LastIndexOf("\\") + 1).Length > 4
                                         where permitNumbers.Contains(f.Substring(f.LastIndexOf("\\") + 1, 5))
                                         select new fileStatus(fileType.DrillerLog, f.Substring(f.LastIndexOf("\\") + 1, 5), 1, f.Substring(f.LastIndexOf("\\") + 1)));

                        permitNumbersWithMatches = (from x in matchingFiles
                                                       select x.PermitNumber);

                        nonMatchingFiles = (from p in permitNumbers
                                                where !permitNumbersWithMatches.Contains(p)
                                            select new fileStatus(fileType.DrillerLog, p, 0, string.Empty));


                        results.AddRange(matchingFiles);
                        results.AddRange(nonMatchingFiles);

                        break;
                    case fileType.RasterLog:

                        matchingFiles = (from f in files
                                         where f.Substring(f.LastIndexOf("\\") + 1).Length > 13
                                         where permitNumbers.Contains(f.Substring(f.LastIndexOf("\\") + 1, 14))
                                         select new fileStatus(fileType.RasterLog, f.Substring(f.LastIndexOf("\\") + 1, 14), 1, f.Substring(f.LastIndexOf("\\") + 1)));

                        permitNumbersWithMatches = (from x in matchingFiles
                                                       select x.PermitNumber);

                        nonMatchingFiles = (from p in permitNumbers
                                                where !permitNumbersWithMatches.Contains(p)
                                            select new fileStatus(fileType.RasterLog, p, 0, string.Empty));



                        results.AddRange(matchingFiles);
                        results.AddRange(nonMatchingFiles);
                        break;
                    default:
                        break;
                }
                log.Debug("Done checking for matching files");
            }
        }
        return results;

    }

一旦到达提供“ matchingFiles”值的linq查询，它就会挂起。 这带有大量的“ permitNumbers”（如5000）和大量的“文件”。

我有什么办法可以加快速度？

考虑到下面提供的建议，我对该功能进行了如下修改，现在性能按预期工作。 非常感谢大家！ =）

public List<fileStatus> checkFilesStatus(List<string> permitNumbers, string serverDirectory, fileType type)
    {
        HashSet<string> numbers = new HashSet<string>(permitNumbers);
        XmlConfigurator.Configure();
        log.Debug(string.Format("Beginning checkFilesStatus with following parameters > permitNumbers: {0} > serverDirectory: {1} > type: {2}", string.Join(",", permitNumbers.ToArray()), serverDirectory, type.ToString()));
        List<fileStatus> results = new List<fileStatus>();
        DirectoryInfo dirInfo = new DirectoryInfo(serverDirectory);
        if (dirInfo.Exists)
        {
            // GET LIST OF ALL FILES IN DIRECTORY
            string[] files = System.IO.Directory.GetFiles(serverDirectory, "*", System.IO.SearchOption.AllDirectories);
            HashSet<string> fileNames = new HashSet<string>(files.Select(f => Path.GetFileName(f)));

            log.Debug(string.Format("List of all files in directory: {0}", string.Join(",", files)));


            if (fileNames.Count > 0 && numbers.Count > 0)
            {
                log.Debug("Checking for matching files");
                // CHECK FOR MATCHING FILES
                switch (type)
                {
                    case fileType.Well:
                        var matchingFiles = (from f in fileNames
                                             where f.Length > 4
                                             where numbers.Contains(f.Substring(0, 5))
                                             select new fileStatus(fileType.Well, f.Substring(0, 5), 1, f));


                        var permitNumbersWithMatches = (from x in matchingFiles
                                                       select x.PermitNumber);

                        var nonMatchingFiles = numbers.Except(permitNumbersWithMatches)
                            .Select(p => new fileStatus(fileType.Well, p, 0, string.Empty));

                        results.AddRange(matchingFiles);
                        results.AddRange(nonMatchingFiles);

                        break;
                    case fileType.DrillerLog:
                        matchingFiles = (from f in fileNames
                                         where f.Length > 4
                                         where numbers.Contains(f.Substring(0, 5))
                                         select new fileStatus(fileType.DrillerLog, f.Substring(0, 5), 1, f));


                        permitNumbersWithMatches = (from x in matchingFiles
                                                       select x.PermitNumber);

                        nonMatchingFiles = numbers.Except(permitNumbersWithMatches)
                            .Select(p => new fileStatus(fileType.DrillerLog, p, 0, string.Empty));


                        results.AddRange(matchingFiles);
                        results.AddRange(nonMatchingFiles);

                        break;
                    case fileType.RasterLog:

                        matchingFiles = (from f in fileNames
                                         where f.Length > 13
                                         where numbers.Contains(f.Substring(0, 14))
                                         select new fileStatus(fileType.RasterLog, f.Substring(0, 14), 1, f));

                        permitNumbersWithMatches = (from x in matchingFiles
                                                       select x.PermitNumber);

                        nonMatchingFiles = numbers.Except(permitNumbersWithMatches)
                            .Select(p => new fileStatus(fileType.RasterLog, p, 0, string.Empty));


                        results.AddRange(matchingFiles);
                        results.AddRange(nonMatchingFiles);
                        break;
                    default:
                        break;
                }
                log.Debug("Done checking for matching files");
            }
        }
        return results;

    }

Answer 1

您正在创建一个matchingFiles 查询 ，当对其进行迭代时，它将遍历您拥有的所有文件，以几种方式对其进行操作，并且还会对数字集进行线性搜索 。 然后，您可以执行此查询并执行该查询（要求重复从磁盘读取大量数据，如果您有足够的钱来破坏高速缓存，那么这将非常昂贵），并对每个许可号执行线性搜索 。 这导致O（N ^ 2 * M）的渐近复杂度，其中N是许可编号的数量，M是文件数量。 那太坏了

此处的关键是避免1）进行线性搜索和2）多次重复复杂的查询，尤其是避免按其他顺序对每个项目进行迭代。

对于＃1，只需将permitNumbers为HashSet<string>而不是列表，然后检查其中是否包含项将成为O（1）操作。

对于＃2，将您的第三个查询替换为只需要迭代一次源序列的操作：

var nonMatchingFiles = permitNumbers.Except(permitNumbersWithMatches)
    .Select(p => new fileStatus(fileType.Well, p, 0, string.Empty));

Answer 2

我将通过对Path.GetFileName（f）的一次调用来消除对f.Substring（f.LastIndexOf（“ \\”）+ 1））的所有重复调用。

例如

var fileNames = files.Select(f => Path.GetFileName(f));    

var matchingFiles = (from fname in fileNames
                     where fname.Length > 4
                     where permitNumbers.Contains(fname.Substring(0, 5))
                     select new fileStatus(fileType.Well, fname.Substring(0, 5), 1, fname);