[英]How do I manage locks when implementing parallel task invocations with Task.WhenAll and max degree of parallelism?
我想出了以下代码,它以 5 的页面大小重复调用数据库分页 function,并且对于页面中的每个项目执行 function 并行最大并发为 4。它看起来像它到目前为止工作,但我不确定我是否需要使用锁定来包含parallelInvocationTasks.Remove(completedTask);
行和Task.WhenAll(parallelInvocationTasks.ToArray());
那么我需要在这里使用锁定吗?您是否看到任何其他改进?
这是代码
程序.cs
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace ConsoleApp1
{
class Program
{
private static async Task Main(string[] args)
{
Console.WriteLine("Starting");
Func<int, int, CancellationToken, Task<IList<string>>> getNextPageFunction = GetNextPageFromDatabase;
await getNextPageFunction.ForEachParallel(4, 5, new CancellationToken(), async (item) =>
{
Console.WriteLine($"{item} started");
//simulate processing
await Task.Delay(1000);
Console.WriteLine($"{item} ended");
});
Console.WriteLine("Done");
}
private static async Task<IList<string>> GetNextPageFromDatabase(
int offset,
int pageSize,
CancellationToken cancellationToken)
{
//simulate i/o and database paging
await Task.Delay(2000, cancellationToken);
var pageData = new List<string>();
//simulate just 4 pages
if (offset >= pageSize * 3)
{
return pageData;
}
for (var i = 1; i <= pageSize; i++)
{
string nextItem = $"Item {i + offset}";
pageData.Add(nextItem);
}
return pageData;
}
}
}
PagingExtensions.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace ConsoleApp1
{
public static class PagingExtensions
{
public static async Task<int> ForEachParallel<TItem>(
this Func<int, int, CancellationToken, Task<IList<TItem>>> getNextPageFunction,
int concurrency,
int pageSize,
CancellationToken cancellationToken,
Func<TItem, Task> forEachFunction)
{
var enumeratedCount = 0;
if (getNextPageFunction == null || forEachFunction == null)
{
return enumeratedCount;
}
var offset = 0;
using (var semaphore = new SemaphoreSlim(concurrency))
{
IList<Task> parallelInvocationTasks = new List<Task>();
IList<TItem> items;
do
{
items = await getNextPageFunction(offset, pageSize, cancellationToken) ?? new List<TItem>();
foreach (TItem item in items)
{
await semaphore.WaitAsync(cancellationToken);
Task forEachFunctionTask = Task.Factory.StartNew(async () =>
{
try
{
await forEachFunction(item);
}
finally
{
// ReSharper disable once AccessToDisposedClosure
// This is safe as long as Task.WhenAll is called before the using semaphore
// enclosure ends
semaphore.Release();
}
}, cancellationToken)
.Unwrap();
parallelInvocationTasks.Add(forEachFunctionTask);
#pragma warning disable 4014
forEachFunctionTask.ContinueWith((completedTask) =>
#pragma warning restore 4014
{
if (completedTask.Exception == null)
{
//Intention is to release completed tasks during enumeration as they complete
//so they can be GCed. This is to ensure the 'parallelInvocationTasks' list does not
//grow in an unmanaged manner resulting in a list holding multiple completed tasks
//unnecessarily consuming more memory with each added invocation task
//Thus the final Task.WhenAll call below will only need to await only faulted tasks
//causing it to throw an exception and/or a minimal list of incomplete tasks only
parallelInvocationTasks.Remove(completedTask);
}
}, cancellationToken);
enumeratedCount += 1;
}
offset += pageSize;
}
while (items.Count >= pageSize);
await Task.WhenAll(parallelInvocationTasks.ToArray());
}
return enumeratedCount;
}
}
}
好的,基于上面的评论和更多的研究,我得出了这个答案,它可以完成工作,而无需编写自定义代码来管理并发性。 它使用来自 TPL DataFlow 的 ActionBlock
PagingExtensions.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
namespace ConsoleApp1
{
public static class PagingExtensions
{
public delegate Task<IList<TItem>> GetNextPageDelegate<TItem>(
int offset,
int pageSize,
CancellationToken cancellationToken);
public static async Task<int> EnumerateParallel<TItem>(
this GetNextPageDelegate<TItem> getNextPageFunction,
int maxDegreeOfParallelism,
int pageSize,
CancellationToken cancellationToken,
Func<TItem, Task> forEachFunction)
{
var enumeratedCount = 0;
if (getNextPageFunction == null || forEachFunction == null)
{
return enumeratedCount;
}
var offset = 0;
var forEachFunctionBlock = new ActionBlock<TItem>(forEachFunction, new ExecutionDataflowBlockOptions
{
BoundedCapacity = pageSize > maxDegreeOfParallelism ? pageSize : maxDegreeOfParallelism,
EnsureOrdered = false,
MaxDegreeOfParallelism = maxDegreeOfParallelism,
CancellationToken = cancellationToken
});
IList<TItem> items;
do
{
items = await getNextPageFunction(offset, pageSize, cancellationToken) ?? new List<TItem>();
foreach (TItem item in items)
{
await forEachFunctionBlock.SendAsync(item, cancellationToken);
enumeratedCount += 1;
}
offset += pageSize;
}
while (items.Count >= pageSize);
forEachFunctionBlock.Complete();
await forEachFunctionBlock.Completion;
return enumeratedCount;
}
}
}
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.