合并多个 IAsyncEnumerable 流

Question

随着Mediatr 10的发布，现在有一个范式允许开发人员创建由IAsyncEnumerable提供支持的流。 我正在利用这种范例来创建多个不同的文件系统观察程序来监视多个文件夹。 为了监控文件夹，我使用了两种不同的方法：轮询和FileSystemWatcher 。 作为我管道的一部分，所有不同的文件夹监视器都聚合到一个IEnumerable<IAsyncEnumerable<FileRecord>中。 在每种类型的观察者中，都有一个内部循环运行，直到通过CancellationToken请求取消。

这是投票观察者：

public class PolledFileStreamHandler : 
    IStreamRequestHandler<PolledFileStream, FileRecord>
{
    private readonly ISeenFileStore _seenFileStore;
    private readonly IPublisher _publisher;
    private readonly ILogger<PolledFileStreamHandler> _logger;

    public PolledFileStreamHandler(
        ISeenFileStore seenFileStore, 
        IPublisher publisher, 
        ILogger<PolledFileStreamHandler> logger)
    {
        _seenFileStore = seenFileStore;
        _publisher = publisher;
        _logger = logger;
    }

    public async IAsyncEnumerable<FileRecord> Handle(
        PolledFileStream request, 
        [EnumeratorCancellation] CancellationToken cancellationToken)
    {
        var queue = new ConcurrentQueue<FileRecord>();
        while (!cancellationToken.IsCancellationRequested)
        {
            var files = Directory.EnumerateFiles(request.Folder)
                .Where(f => !_seenFileStore.Contains(f));

            await Parallel.ForEachAsync(files, CancellationToken.None, async (f,t) =>
            {
                var info = new FileRecord(f);
                
                _seenFileStore.Add(f);
                await _publisher.Publish(new FileSeenNotification { FileInfo = info }, t);
                queue.Enqueue(info);
            });
            
            // TODO: Try mixing the above parallel task with the serving task... Might be chaos...

            while (!queue.IsEmpty)
            {
                if (queue.TryDequeue(out var result))
                    yield return result;
            }

            _logger.LogInformation("PolledFileStreamHandler watching {Directory} at: {Time}", request.Folder, DateTimeOffset.Now);
            
            await Task.Delay(request.Interval, cancellationToken)
                .ContinueWith(_ => {}, CancellationToken.None);
        }
    }
}

和 FileSystemWatcher

public class FileSystemStreamHandler : 
    IStreamRequestHandler<FileSystemStream, FileRecord>
{
    private readonly ISeenFileStore _seenFileStore;
    private readonly ILogger<FileSystemStreamHandler> _logger;
    private readonly IPublisher _publisher;
    private readonly ConcurrentQueue<FileRecord> _queue;

    private Action<object, FileSystemEventArgs>? _tearDown;

    public FileSystemStreamHandler(
        ISeenFileStore seenFileStore, 
        ILogger<FileSystemStreamHandler> logger, 
        IPublisher publisher)
    {
        _seenFileStore = seenFileStore;
        _logger = logger;
        _publisher = publisher;
        _queue = new ConcurrentQueue<FileRecord>();
    }

    public async IAsyncEnumerable<FileRecord> Handle(
        FileSystemStream request, 
        [EnumeratorCancellation] CancellationToken cancellationToken)
    {
        var watcher = SetupWatcher(request.Folder, cancellationToken);
        
        while (!cancellationToken.IsCancellationRequested)
        {
            if (_queue.TryDequeue(out var record))
                yield return record;

            await Task.Delay(100, cancellationToken)
                .ContinueWith(_ => {}, CancellationToken.None);
        }
        
        TearDownWatcher(watcher);
    }
    
    private FileSystemWatcher SetupWatcher(string folder, CancellationToken cancellation)
    {
        var watcher = new FileSystemWatcher(folder);
        watcher.NotifyFilter = NotifyFilters.Attributes
                               | NotifyFilters.CreationTime
                               | NotifyFilters.DirectoryName
                               | NotifyFilters.FileName
                               | NotifyFilters.LastAccess
                               | NotifyFilters.LastWrite
                               | NotifyFilters.Security
                               | NotifyFilters.Size;
        watcher.EnableRaisingEvents = true;
        _tearDown = (_, args) => OnWatcherOnChanged(args, cancellation);
        watcher.Created += _tearDown.Invoke;

        return watcher;
    }
    
    private async void OnWatcherOnChanged(FileSystemEventArgs args, CancellationToken cancellationToken)
    {
        var path = args.FullPath;

        if (_seenFileStore.Contains(path)) return;
            
        _seenFileStore.Add(path);

        try
        {
            if ((File.GetAttributes(path) & FileAttributes.Directory) != 0) return;
        }
        catch (FileNotFoundException)
        {
            _logger.LogWarning("File {File} was not found. During a routine check. Will not be broadcast", path);
            return;
        }
            
        var record = new FileRecord(path);
        _queue.Enqueue(record);
        await _publisher.Publish(new FileSeenNotification { FileInfo = record }, cancellationToken);
    }

    private void TearDownWatcher(FileSystemWatcher watcher)
    {
        if (_tearDown != null)
            watcher.Created -= _tearDown.Invoke;
    }
}

最后，这是 class 将所有内容联系在一起并尝试监视流（在StartAsync方法中）。 您会注意到来自System.Interactive.Async的Merge运算符的存在，这当前无法按预期运行。

public class StreamedFolderWatcher : IDisposable
{
    private readonly ConcurrentBag<Func<IAsyncEnumerable<FileRecord>>> _streams;
    private CancellationTokenSource? _cancellationTokenSource;
    private readonly IMediator _mediator;
    private readonly ILogger<StreamedFolderWatcher> _logger;

    public StreamedFolderWatcher(
        IMediator mediator,
        IEnumerable<IFileStream> fileStreams, 
        ILogger<StreamedFolderWatcher> logger)
    {
        _mediator = mediator;
        _logger = logger;
        _streams = new ConcurrentBag<Func<IAsyncEnumerable<FileRecord>>>();
        _cancellationTokenSource = new CancellationTokenSource();

        fileStreams.ToList()
            .ForEach(f => AddStream(f, _cancellationTokenSource.Token));
    }

    private void AddStream<T>(
        T request, 
        CancellationToken cancellationToken) 
        where T : IStreamRequest<FileRecord>
    {
        _streams.Add(() => _mediator.CreateStream(request, cancellationToken));
    }

    public async Task StartAsync(CancellationToken cancellationToken)
    {
        _cancellationTokenSource = CancellationTokenSource
            .CreateLinkedTokenSource(cancellationToken);

        var streams = _streams.Select(s => s()).ToList();
        while (!cancellationToken.IsCancellationRequested)
        {
            await foreach (var file in streams.Merge().WithCancellation(cancellationToken))
            {
                _logger.LogInformation("Incoming file {File}", file);
            }
            
            await Task.Delay(1000, cancellationToken)
                .ContinueWith(_ => {}, CancellationToken.None);
        }
    }

    public Task StopAsync()
    {
        _cancellationTokenSource?.Cancel();

        return Task.CompletedTask;
    }

    public void Dispose()
    {
        _cancellationTokenSource?.Dispose();
        GC.SuppressFinalize(this);
    }
}

我对Merge行为的期望是，如果我有 3 个IAsyncEnumerables ，则每个项目应在产生后立即发出。 相反，除非我在循环中的某处放置yield break ，否则获取的第一个IStreamRequestHandler将简单地无限执行直到取消令牌强制停止。

如何将多个输入IAsyncEnumerables合并到一个长寿命的 output stream 中，每次产生结果时都会发出？

最小可重现样品

static async IAsyncEnumerable<(Guid Id, int Value)> CreateSequence(
    [EnumeratorCancellation] CancellationToken cancellationToken)
{
    var random = new Random();
    var id = Guid.NewGuid();
    while (!cancellationToken.IsCancellationRequested)
    {
        await Task.Delay(TimeSpan.FromMilliseconds(random.Next(100, 1000)));
        yield return (id, random.Next(0, 10));
    }
}

var token = new CancellationTokenSource();
var sequences = Enumerable.Range(0, 10)
    .Select(_ => CreateSequence(token.Token));
var merged = sequences.Merge();

await foreach (var (id, value) in merged)
{
    Console.WriteLine($"[{DateTime.Now.ToShortTimeString()}] Value {value} Emitted from {id}");
}

Answer 1

我设法提出了一个可行的，但可能效率低下且可能存在错误的解决方案。 通过将每个IAsyncEnumerable放入它自己的后台任务中，我可以将每个 IAsyncEnumerable 发送到一个线程安全队列中，当每个可用时，它们都会在其中提供服务。

public static async IAsyncEnumerable<TSource> MergeAsyncEnumerable<TSource>(
    this IList<IAsyncEnumerable<TSource>> sources,
    [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
    var queue = new ConcurrentQueue<TSource>();
    while (!cancellationToken.IsCancellationRequested)
    {
        var collections = sources
            .Select(s => Task.Run(async () =>
            {
                await foreach (var file in s.WithCancellation(cancellationToken))
                {
                    queue.Enqueue(file);
                }
            }, cancellationToken))
            .ToList();

        while (!Task.WhenAll(collections).IsCompleted)
        {
            while (!queue.IsEmpty)
                if (queue.TryDequeue(out var record))
                    yield return record;
            
            await Task.Delay(100, cancellationToken)
                .ContinueWith(_ => {}, CancellationToken.None);    
        }
    }
}

Answer 2

这是我试图重现您的观察结果的尝试。 我创建了两个异步序列，每个序列包含 5 个值，以 200 毫秒的间隔发出。 然后我将它们与AsyncEnumerableEx.Merge运算符合并，最后我枚举了合并后的序列：

var sequence1 = CreateSequence(1, 2, 3, 4, 5);
var sequence2 = CreateSequence(11, 12, 13, 14, 15);
var merged = AsyncEnumerableEx.Merge(sequence1, sequence2);

await foreach (var item in merged) Print(item);

static async IAsyncEnumerable<int> CreateSequence(params int[] values)
{
    foreach (var value in values)
    {
        await Task.Delay(200); yield return value;
    }
}

static void Print(object value)
{
    Console.WriteLine($@"{DateTime.Now:HH:mm:ss.fff} [{Thread.CurrentThread
        .ManagedThreadId}] > {value}");
}

Output：

00:01:28.337 [4] > 11
00:01:28.362 [4] > 1
00:01:28.552 [4] > 12
00:01:28.564 [4] > 2
00:01:28.767 [4] > 3
00:01:28.769 [7] > 13
00:01:28.971 [7] > 14
00:01:28.972 [7] > 4
00:01:29.174 [7] > 5
00:01:29.177 [7] > 15

在 Fiddle 上试试。

简而言之，我无法重现您的观察结果。 合并后的序列似乎会及时发出两个源序列的元素。