[英]How to download multiple files at once from S3 using C# AWS SDK
如何從 s3 存儲桶下載多個文件。 我在 SO 上找不到更好的選擇。 這是我的單個文件下載代碼。 給定 Urls 列表,我正在循環下載多個文件。
public async Task Download(string url, Stream output)
{
var s3Uri = new AmazonS3Uri(url);
GetObjectRequest getObjectRequest = new GetObjectRequest
{
BucketName = s3Uri.Bucket,
Key = System.Net.WebUtility.UrlDecode(s3Uri.Key)
};
using (var s3Client = new AmazonS3Client(s3Uri.Region))
{
// dispose the underline stream when writing to stream is done
using (var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest).ConfigureAwait(false))
{
using (var responseStream = getObjectResponse.ResponseStream)
{
await responseStream.CopyToAsync(output);
}
}
}
output.Seek(0L, SeekOrigin.Begin);
}
下載給定 s3 url 的文件
var list = new List<Stream>();
foreach(var url in urls)
{
var stream = new MemoryStream();
await Download(url,ms);
list.Add(stream);
}
從 S3 一次下載多個文件是否有更好的選擇?
我終於決定實現我自己的版本
public class StreamWrapper
{
public string Url { get; set; }
public Stream Content { get; set; }
public string FileName { get; set; }
}
public async Task Download(IList<StreamWrapper> inout, int maxConcurrentDownloads)
{
if (maxConcurrentDownloads <= 0)
{
maxConcurrentDownloads = 20;
}
if (!inout.HasAny())
return;
var tasks = new List<Task>();
for (int i = 0; i < inout.Count; i++)
{
StreamWrapper wrapper = inout[i];
AmazonS3Uri s3Uri = null;
if (AmazonS3Uri.TryParseAmazonS3Uri(wrapper.Url, out s3Uri))
{
tasks.Add(GetObject(s3Uri, wrapper.Content));
}
if (tasks.Count == maxConcurrentDownloads || i == inout.Count - 1)
{
await Task.WhenAll(tasks);
tasks.Clear();
}
}
}
private async Task GetObject(AmazonS3Uri s3Uri, Stream output)
{
GetObjectRequest getObjectRequest = new GetObjectRequest
{
BucketName = s3Uri.Bucket,
Key = System.Net.WebUtility.UrlDecode(s3Uri.Key)
};
using (var s3Client = new AmazonS3Client(s3Uri.Region))
{
// dispose the underline stream when writing to local file system is done
using (var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest).ConfigureAwait(false))
{
using (var responseStream = getObjectResponse.ResponseStream)
{
await responseStream.CopyToAsync(output);
}
}
}
output.Seek(0L, SeekOrigin.Begin);
}
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.