使用 AWS Lambda 从 S3 上的目录创建 Tar 存档

Question

我需要提取一堆存储在 s3 上的 zip 文件，并将它们添加到 tar 存档并将该存档存储在 s3 上。 zip 文件的总和可能会大于 lambda 函数允许的 512mb 本地存储。 我有一个部分souldtion，它从s3 中提取对象并将它们放入s3 对象中，而不使用lambda 本地存储。

提取对象线程

public class ExtractObject implements Runnable{

    private String objectName;
    private String uuid;
    private final byte[] buffer = new byte[1024];

    public ExtractAdvert(String name, String uuid) {
        this.objectName= name;
        this.uuid= uuid;
    }

    @Override
    public void run() {
        final String srcBucket = "my-bucket-name";
        final AmazonS3 s3Client = new AmazonS3Client();

        try {
            S3Object s3Object = s3Client.getObject(new GetObjectRequest(srcBucket, objectName));
            ZipInputStream zis = new ZipInputStream(s3Object.getObjectContent());
            ZipEntry entry = zis.getNextEntry();

            while(entry != null) {
                String fileName = entry.getName();
                String mimeType = FileMimeType.fromExtension(FilenameUtils.getExtension(fileName)).mimeType();
                System.out.println("Extracting " + fileName + ", compressed: " + entry.getCompressedSize() + " bytes, extracted: " + entry.getSize() + " bytes, mimetype: " + mimeType);
                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
                int len;
                while ((len = zis.read(buffer)) > 0) {
                    outputStream.write(buffer, 0, len);
                }
                InputStream is = new ByteArrayInputStream(outputStream.toByteArray());
                ObjectMetadata meta = new ObjectMetadata();
                meta.setContentLength(outputStream.size());
                meta.setContentType(mimeType);
                System.out.println("##### " + srcBucket + ", " + FilenameUtils.getFullPath(objectName) + "tmp" + File.separator + uuid + File.separator + fileName);

                // Add this to tar archive instead of putting back to s3
                s3Client.putObject(srcBucket, FilenameUtils.getFullPath(objectName) + "tmp" + File.separator + uuid + File.separator + fileName, is, meta);
                is.close();
                outputStream.close();
                entry = zis.getNextEntry();
            }
            zis.closeEntry();
            zis.close();
        } catch (IOException ioe) {
                System.out.println(ioe.getMessage());
        }
    }
}

---

这会针对需要提取的每个对象运行，并将它们保存在 tar 文件所需结构中的 s3 对象中。

我认为我需要的是将对象保存在内存中并将其添加到 tar 存档中，而不是将对象放回 s3。 并上传它，但经过多次环顾和反复试验，我还没有创建成功的 tar 文件。 主要问题是我无法在 lambda 中使用 tmp 目录。

---

编辑我应该创建 tar 文件而不是将对象放入 s3 吗？ （请参阅评论// Add this to tar archive instead of putting back to s3 ）如果是这样，我如何创建 tar 流而不将其存储在本地？

---

编辑 2：尝试去皮重文件

ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
ListObjectsV2Result result;

ByteArrayOutputStream baos = new ByteArrayOutputStream();
TarArchiveOutputStream tarOut = new TarArchiveOutputStream(baos);

do {
    result = s3Client.listObjectsV2(req);

    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {

        if(objectSummary.getKey().startsWith("tmp/") )  {
            System.out.printf(" - %s (size: %d)\n", objectSummary.getKey(), objectSummary.getSize());
            S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectSummary.getKey()));
            InputStream is = s3Object.getObjectContent(); 
            System.out.println("Pre Create entry");
            TarArchiveEntry archiveEntry = new TarArchiveEntry(IOUtils.toByteArray(is));
            // Getting following exception above
            // IllegalArgumentException: Invalid byte 111 at offset 7 in ' positio' len=8
            System.out.println("Pre put entry");
            tarOut.putArchiveEntry(archiveEntry);
            System.out.println("Post put entry");
        }
    }

    String token = result.getNextContinuationToken();
    System.out.println("Next Continuation Token: " + token);
    req.setContinuationToken(token);
} while (result.isTruncated());

ObjectMetadata metadata = new ObjectMetadata();
InputStream is = new ByteArrayInputStream(baos.toByteArray());
s3Client.putObject(new PutObjectRequest(bucketName, bucketFolder + "tar-file", is, metadata));

Answer 1

我找到了一个解决方案，它与我在上面的Edit 2 中的尝试非常相似。

private final String bucketName = "bucket-name";
private final String bucketFolder = "tmp/";
private final String tarKey = "tar-dir/tared-file.tar";

private void createTar() throws IOException, ArchiveException {
    ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
    ListObjectsV2Result result;

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    TarArchiveOutputStream tarOut = new TarArchiveOutputStream(baos);

    do {
        result = s3Client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            if (objectSummary.getKey().startsWith(bucketFolder)) {
                S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectSummary.getKey()));
                InputStream is = s3Object.getObjectContent();

                String s3Key = objectSummary.getKey();
                String tarPath = s3Key.substring(s3Key.indexOf('/') + 1, s3Key.length());
                s3Key.lastIndexOf('.'));

                byte[] ba = IOUtils.toByteArray(is);

                TarArchiveEntry archiveEntry = new TarArchiveEntry(tarPath);
                archiveEntry.setSize(ba.length);
                tarOut.putArchiveEntry(archiveEntry);
                tarOut.write(ba);
                tarOut.closeArchiveEntry();
            }
        }

        String token = result.getNextContinuationToken();
        System.out.println("Next Continuation Token: " + token);
        req.setContinuationToken(token);
    } while (result.isTruncated());

    ObjectMetadata metadata = new ObjectMetadata();
    InputStream is = baos.toInputStream();
    metadata.setContentLength(baos.size());
    s3Client.putObject(new PutObjectRequest(bucketName, tarKey, is, metadata));
}