使用 NodeJS 將文件上傳到 Amazon S3

Question

我在嘗試將文件上傳到我的 S3 存儲桶時遇到了問題。 一切正常，除了我的文件參數似乎不合適。 我正在使用 Amazon S3 sdk 從 nodejs 上傳到 s3。

這些是我的路線設置：

var multiparty = require('connect-multiparty'),
    multipartyMiddleware = multiparty();
app.route('/api/items/upload').post(multipartyMiddleware, items.upload);

這是 items.upload() 函數：

exports.upload = function(req, res) {
    var file = req.files.file;
    var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
    s3bucket.createBucket(function() {
        var params = {
            Key: file.name,
            Body: file
        };
        s3bucket.upload(params, function(err, data) {
            console.log("PRINT FILE:", file);
            if (err) {
                console.log('ERROR MSG: ', err);
            } else {
                console.log('Successfully uploaded data');
            }
        });
    });
};

將Body參數設置為"hello"類的字符串可以正常工作。 根據doc ， Body參數必須采用(Buffer, Typed Array, Blob, String, ReadableStream) 對象數據。 但是，上傳文件對象失敗並顯示以下錯誤消息：

[Error: Unsupported body payload object]

這是文件對象：

{ fieldName: 'file',
  originalFilename: 'second_fnp.png',
  path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
  headers: 
   { 'content-disposition': 'form-data; name="file"; filename="second_fnp.png"',
     'content-type': 'image/png' },
  ws: 
   { _writableState: 
      { highWaterMark: 16384,
        objectMode: false,
        needDrain: true,
        ending: true,
        ended: true,
        finished: true,
        decodeStrings: true,
        defaultEncoding: 'utf8',
        length: 0,
        writing: false,
        sync: false,
        bufferProcessing: false,
        onwrite: [Function],
        writecb: null,
        writelen: 0,
        buffer: [],
        errorEmitted: false },
     writable: true,
     domain: null,
     _events: { error: [Object], close: [Object] },
     _maxListeners: 10,
     path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
     fd: null,
     flags: 'w',
     mode: 438,
     start: undefined,
     pos: undefined,
     bytesWritten: 261937,
     closed: true },
  size: 261937,
  name: 'second_fnp.png',
  type: 'image/png' }

任何幫助將不勝感激！

Answer 1

所以看起來這里有一些問題。 根據您的帖子，您似乎正在嘗試使用connect-multiparty中間件支持文件上傳。 這個中間件的作用是獲取上傳的文件，將其寫入本地文件系統，然后將req.files設置為上傳的文件。

您的路線配置看起來不錯，問題似乎出在您的items.upload()函數上。 特別是這部分：

var params = {
  Key: file.name,
  Body: file
};

正如我在回答開頭提到的， connect-multiparty將文件寫入本地文件系統，因此您需要打開文件並讀取它，然后上傳它，然后在本地文件系統上將其刪除。

也就是說，您可以將方法更新為如下所示：

var fs = require('fs');
exports.upload = function (req, res) {
    var file = req.files.file;
    fs.readFile(file.path, function (err, data) {
        if (err) throw err; // Something went wrong!
        var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
        s3bucket.createBucket(function () {
            var params = {
                Key: file.originalFilename, //file.name doesn't exist as a property
                Body: data
            };
            s3bucket.upload(params, function (err, data) {
                // Whether there is an error or not, delete the temp file
                fs.unlink(file.path, function (err) {
                    if (err) {
                        console.error(err);
                    }
                    console.log('Temp File Delete');
                });

                console.log("PRINT FILE:", file);
                if (err) {
                    console.log('ERROR MSG: ', err);
                    res.status(500).send(err);
                } else {
                    console.log('Successfully uploaded data');
                    res.status(200).end();
                }
            });
        });
    });
};

這樣做是從本地文件系統讀取上傳的文件，然后將其上傳到 S3，然后刪除臨時文件並發送響應。

這種方法存在一些問題。 首先，它的效率並不高，因為對於大文件，您將在寫入之前加載整個文件。 其次，這個過程不支持大文件的分段上傳（我認為在你必須進行分段上傳之前截止是 5 Mb）。

相反，我建議您使用我一直在研究的名為S3FS的模塊，它提供與Node.JS中的本機FS類似的接口，但抽象了一些細節，例如分段上傳和 S3 api （以及添加一些附加功能，如遞歸方法）。

如果您要引入 S3FS庫，您的代碼將如下所示：

var fs = require('fs'),
    S3FS = require('s3fs'),
    s3fsImpl = new S3FS('mybucketname', {
        accessKeyId: XXXXXXXXXXX,
        secretAccessKey: XXXXXXXXXXXXXXXXX
    });

// Create our bucket if it doesn't exist
s3fsImpl.create();

exports.upload = function (req, res) {
    var file = req.files.file;
    var stream = fs.createReadStream(file.path);
    return s3fsImpl.writeFile(file.originalFilename, stream).then(function () {
        fs.unlink(file.path, function (err) {
            if (err) {
                console.error(err);
            }
        });
        res.status(200).end();
    });
};

這將做的是實例化提供的存儲桶和 AWS 憑證的模塊，然后創建存儲桶（如果它不存在）。 然后，當請求上傳文件時，我們將打開文件的流並使用它將文件寫入 S3 的指定路徑。 這將在幕后處理多部分上傳片段（如果需要），並具有通過流完成的好處，因此您不必在開始上傳之前等待讀取整個文件。

如果您願意，可以將代碼更改為Promises 的回調。 或者使用帶有事件偵聽器的pipe()方法來確定結束/錯誤。

如果您正在尋找一些其他方法，請查看 s3fs 的文檔，如果您正在尋找一些其他方法或遇到問題，請隨時提出問題。

Answer 2

我發現以下是一個有效的解決方案：

npm install aws-sdk

安裝 aws-sdk 后，請使用以下代碼在需要的地方替換值。

var AWS = require('aws-sdk');
var fs =  require('fs');

var s3 = new AWS.S3();

// Bucket names must be unique across all S3 users

var myBucket = 'njera';

var myKey = 'jpeg';
//for text file
//fs.readFile('demo.txt', function (err, data) {
//for Video file
//fs.readFile('demo.avi', function (err, data) {
//for image file                
fs.readFile('demo.jpg', function (err, data) {
  if (err) { throw err; }



     params = {Bucket: myBucket, Key: myKey, Body: data };

     s3.putObject(params, function(err, data) {

         if (err) {

             console.log(err)

         } else {

             console.log("Successfully uploaded data to myBucket/myKey");

         }

      });

});

我在這里找到了有關該主題的完整教程，以防您正在尋找參考資料 ::

---

如何使用 node.js 在亞馬遜 s3 中上傳文件（文本/圖像/視頻）

Answer 3

或使用承諾：

const AWS = require('aws-sdk');
AWS.config.update({
    accessKeyId: 'accessKeyId',
    secretAccessKey: 'secretAccessKey',
    region: 'region'
});

let params = {
    Bucket: "yourBucketName",
    Key: 'someUniqueKey',
    Body: 'someFile'
};
try {
    let uploadPromise = await new AWS.S3().putObject(params).promise();
    console.log("Successfully uploaded data to bucket");
} catch (e) {
    console.log("Error uploading data: ", e);
}

Answer 4

將文件上傳到 AWS s3 並發送 URL 以響應訪問該文件。

Multer 是一個用於處理 multipart/form-data 的 node.js 中間件，主要用於上傳文件。 它寫在 busboy 之上，以實現最高效率。 在這里檢查這個 npm 模塊。

發送請求時，請確保標頭的 Content-Type 為 multipart/form-data。 我們在響應中發送文件位置，它將提供 url，但如果您想訪問該 url，請將存儲桶設為公開，否則您將無法訪問它。

上傳路由器.js

const express = require('express');
const router = express.Router();
const AWS = require('aws-sdk');
const multer = require('multer');
const storage = multer.memoryStorage()
const upload = multer({storage: storage});

const s3Client = new AWS.S3({
    accessKeyId: 'your_access_key_id',
    secretAccessKey: 'your_secret_access_id',
    region :'ur region'
});

const uploadParams = {
         Bucket: 'ur_bucket_name', 
         Key: '', // pass key
         Body: null, // pass file body
};


router.post('/api/file/upload', upload.single("file"),(req,res) => {
    const params = uploadParams;

    uploadParams.Key = req.file.originalname;
    uploadParams.Body = req.file.buffer;

    s3Client.upload(params, (err, data) => {
        if (err) {
            res.status(500).json({error:"Error -> " + err});
        }
        res.json({message: 'File uploaded successfully','filename': 
        req.file.originalname, 'location': data.Location});
    });
});

module.exports = router;

應用程序.js

const express = require('express');
const app = express();

const router = require('./app/routers/upload.router.js');
app.use('/', router);

// Create a Server
  const server = app.listen(8080, () => {
  console.log("App listening at 8080"); 
})

Answer 5

上傳 CSV/Excel

const fs = require('fs');
const AWS = require('aws-sdk');

const s3 = new AWS.S3({
    accessKeyId: XXXXXXXXX,
    secretAccessKey: XXXXXXXXX
});

const absoluteFilePath = "C:\\Project\\test.xlsx";

const uploadFile = () => {
  fs.readFile(absoluteFilePath, (err, data) => {
     if (err) throw err;
     const params = {
         Bucket: 'testBucket', // pass your bucket name
         Key: 'folderName/key.xlsx', // file will be saved in <folderName> folder
         Body: data
     };
      s3.upload(params, function (s3Err, data) {
                    if (s3Err) throw s3Err
                    console.log(`File uploaded successfully at ${data.Location}`);
                    debugger;
                });
  });
};

uploadFile();

Answer 6

對我有用:)

  const fileContent = fs.createReadStream(`${fileName}`);
  return new Promise(function (resolve, reject) {
    fileContent.once('error', reject);
    s3.upload(
      {
        Bucket: 'test-bucket',
        Key: `${fileName + '_' + Date.now().toString()}`,
        ContentType: 'application/pdf',
        ACL: 'public-read',
        Body: fileContent
      },
      function (err, result) {
        if (err) {
          reject(err);
          return;
        }
        resolve(result.Location);
      }
    );
  });```

Answer 7

感謝 David，他的解決方案幫助我提出了將多部分文件從 Heroku 托管站點上傳到 S3 存儲桶的解決方案。 我使用 formidable 來處理傳入的表單和 fs 來獲取文件內容。 希望，它可以幫助你。

api.service.ts

public upload(files): Observable<any> {  
    const formData: FormData = new FormData(); 
    files.forEach(file => {
      // create a new multipart-form for every file 
      formData.append('file', file, file.name);           
    });   
    return this.http.post(uploadUrl, formData).pipe(
      map(this.extractData),
      catchError(this.handleError)); 
  }
}

服務器.js

app.post('/api/upload', upload);
app.use('/api/upload', router);

上傳.js

const IncomingForm = require('formidable').IncomingForm;
const fs = require('fs');
const AWS = require('aws-sdk');

module.exports = function upload(req, res) {
    var form = new IncomingForm();

    const bucket = new AWS.S3(
      {
        signatureVersion: 'v4',
        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
        region: 'us-east-1'       
      }
    ); 

    form.on('file', (field, file) => {

        const fileContent = fs.readFileSync(file.path);

        const s3Params = {
            Bucket: process.env.AWS_S3_BUCKET,
            Key: 'folder/' + file.name,
            Expires: 60,             
            Body: fileContent,
            ACL: 'public-read'
        };

        bucket.upload(s3Params, function(err, data) {
            if (err) {
                throw err;
            }            
            console.log('File uploaded to: ' + data.Location);
            fs.unlink(file.path, function (err) {
              if (err) {
                  console.error(err);
              }
              console.log('Temp File Delete');
          });
        });
    });              

    // The second callback is called when the form is completely parsed. 
    // In this case, we want to send back a success status code.
    form.on('end', () => {        
      res.status(200).json('upload ok');
    });

    form.parse(req);
}

上傳-image.component.ts

import { Component, OnInit, ViewChild, Output, EventEmitter, Input } from '@angular/core';
import { ApiService } from '../api.service';
import { MatSnackBar } from '@angular/material/snack-bar';

@Component({
  selector: 'app-upload-image',
  templateUrl: './upload-image.component.html',
  styleUrls: ['./upload-image.component.css']
})

export class UploadImageComponent implements OnInit {
  public files: Set<File> = new Set();
  @ViewChild('file', { static: false }) file;
  public uploadedFiles: Array<string> = new Array<string>();
  public uploadedFileNames: Array<string> = new Array<string>();
  @Output() filesOutput = new EventEmitter<Array<string>>();
  @Input() CurrentImage: string;
  @Input() IsPublic: boolean;
  @Output() valueUpdate = new EventEmitter();
  strUploadedFiles:string = '';
  filesUploaded: boolean = false;     

  constructor(private api: ApiService, public snackBar: MatSnackBar,) { }

  ngOnInit() {    
  }

  updateValue(val) {  
    this.valueUpdate.emit(val);  
  }  

  reset()
  {
    this.files = new Set();
    this.uploadedFiles = new Array<string>();
    this.uploadedFileNames = new Array<string>();
    this.filesUploaded = false;
  }

  upload() { 

    this.api.upload(this.files).subscribe(res => {   
      this.filesOutput.emit(this.uploadedFiles); 
      if (res == 'upload ok')
      {
        this.reset(); 
      }     
    }, err => {
      console.log(err);
    });
  }

  onFilesAdded() {
    var txt = '';
    const files: { [key: string]: File } = this.file.nativeElement.files;

    for (let key in files) {
      if (!isNaN(parseInt(key))) {

        var currentFile = files[key];
        var sFileExtension = currentFile.name.split('.')[currentFile.name.split('.').length - 1].toLowerCase();
        var iFileSize = currentFile.size;

        if (!(sFileExtension === "jpg" 
              || sFileExtension === "png") 
              || iFileSize > 671329) {
            txt = "File type : " + sFileExtension + "\n\n";
            txt += "Size: " + iFileSize + "\n\n";
            txt += "Please make sure your file is in jpg or png format and less than 655 KB.\n\n";
            alert(txt);
            return false;
        }

        this.files.add(files[key]);
        this.uploadedFiles.push('https://gourmet-philatelist-assets.s3.amazonaws.com/folder/' + files[key].name);
        this.uploadedFileNames.push(files[key].name);
        if (this.IsPublic && this.uploadedFileNames.length == 1)
        {
          this.filesUploaded = true;
          this.updateValue(files[key].name);
          break;
        } 
        else if (!this.IsPublic && this.uploadedFileNames.length == 3)
        {
          this.strUploadedFiles += files[key].name;          
          this.updateValue(this.strUploadedFiles); 
          this.filesUploaded = true;
          break;
        }
        else
        {
          this.strUploadedFiles += files[key].name + ",";          
          this.updateValue(this.strUploadedFiles); 
        }      
      }
    }    
  }

  addFiles() {
    this.file.nativeElement.click();  
  }

  openSnackBar(message: string, action: string) {
    this.snackBar.open(message, action, {
      duration: 2000,
      verticalPosition: 'top'
    });
  }   

}

上傳-image.component.html

<input type="file" #file style="display: none" (change)="onFilesAdded()" multiple />
&nbsp;<button mat-raised-button color="primary" 
         [disabled]="filesUploaded" (click)="$event.preventDefault(); addFiles()">
  Add Files
</button>
&nbsp;<button class="btn btn-success" [disabled]="uploadedFileNames.length == 0" (click)="$event.preventDefault(); upload()">
  Upload
</button>

Answer 8

var express = require('express')

app = module.exports = express();
var secureServer = require('http').createServer(app);
secureServer.listen(3001);

var aws = require('aws-sdk')
var multer = require('multer')
var multerS3 = require('multer-s3')

    aws.config.update({
    secretAccessKey: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
    accessKeyId: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
    region: 'us-east-1'
    });
    s3 = new aws.S3();

   var upload = multer({
   storage: multerS3({
    s3: s3,
    dirname: "uploads",
    bucket: "Your bucket name",
    key: function (req, file, cb) {
        console.log(file);
        cb(null, "uploads/profile_images/u_" + Date.now() + ".jpg"); //use  
     Date.now() for unique file keys
    }
  })
   });

 app.post('/upload', upload.single('photos'), function(req, res, next) {

 console.log('Successfully uploaded ', req.file)

 res.send('Successfully uploaded ' + req.file.length + ' files!')

})

Answer 9

使用 aws SDK v3

npm install @aws-sdk/client-s3

上傳代碼

import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";

/**
* advisable to save your AWS credentials and configurations in an environmet file. Not inside the code
* AWS lib will automatically load the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if available in your environment
*/
const s3Client = new S3Client({ region: process.env.AWS_S3_REGION });

/**
* upload a file
* @param file the file object to be uploaded
* @param fileKey the fileKey. could be separated with '/' to nest the file into a folder structure. eg. members/user1/profile.png
*/
export function uploadFile(file, fileKey){
    s3Client.send(new PutObjectCommand({
       Bucket: process.env.MY_AWS_S3_BUCKET,
       Key: fileKey,
       Body: file
    }));
}

如果你想下載

import { GetObjectCommand } from "@aws-sdk/client-s3";
/**
 * download a file from AWS and send to your rest client
 */
app.get('/download', function(req, res, next){
    var fileKey = req.query['fileKey'];

    var bucketParams = {
        Bucket: 'my-bucket-name',
        Key: fileKey,
    };

    res.attachment(fileKey);
    var fileStream = await s3Client.send(new GetObjectCommand(bucketParams));
    // for TS you can add: if (fileStream.Body instanceof Readable)
    fileStream.Body.pipe(res)
});

Answer 10

在這里您可以找到上述錯誤的解決方案

https://docs.amplify.aws/lib/storage/upload/q/platform/js/#browser-uploads