How to upload files to S3 bucket from Django lambda deployment using Zappa?

Question

I have a basic web application that allows users to upload files. There is no processing involved, I just need them to be stored in a specific Amazon S3 bucket. I have the application deployed using Zappa on AWS Lambda. The document upload functionality works perfectly on the local test server, but I am not sure how to make it work on the actual deployed instance. I get the same error if I directly try to add a file through the admin panel. Does anyone know what I'm doing wrong or how to get around this? I'll include the traceback here:

File "/var/task/django/core/handlers/exception.py" in inner
  34.             response = get_response(request)

File "/var/task/django/core/handlers/base.py" in _get_response
  115.                 response = self.process_exception_by_middleware(e, request)

File "/var/task/django/core/handlers/base.py" in _get_response
  113.                 response = wrapped_callback(request, *callback_args, **callback_kwargs)

File "/var/task/django/views/generic/base.py" in view
  71.             return self.dispatch(request, *args, **kwargs)

File "/var/task/django/contrib/auth/mixins.py" in dispatch
  52.         return super().dispatch(request, *args, **kwargs)

File "/var/task/django/contrib/auth/mixins.py" in dispatch
  85.         return super().dispatch(request, *args, **kwargs)

File "/var/task/django/views/generic/base.py" in dispatch
  97.         return handler(request, *args, **kwargs)

File "/var/task/django/views/generic/edit.py" in post
  172.         return super().post(request, *args, **kwargs)

File "/var/task/django/views/generic/edit.py" in post
  142.             return self.form_valid(form)

File "/var/task/myapp/views.py" in form_valid
  83.         return super(UploadView, self).form_valid(form)

File "/var/task/django/views/generic/edit.py" in form_valid
  125.         self.object = form.save()

File "/var/task/django/forms/models.py" in save
  458.             self.instance.save()

File "/var/task/django/db/models/base.py" in save
  741.                        force_update=force_update, update_fields=update_fields)

File "/var/task/django/db/models/base.py" in save_base
  779.                 force_update, using, update_fields,

File "/var/task/django/db/models/base.py" in _save_table
  870.             result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)

File "/var/task/django/db/models/base.py" in _do_insert
  908.                                using=using, raw=raw)

File "/var/task/django/db/models/manager.py" in manager_method
  82.                 return getattr(self.get_queryset(), name)(*args, **kwargs)

File "/var/task/django/db/models/query.py" in _insert
  1186.         return query.get_compiler(using=using).execute_sql(return_id)

File "/var/task/django/db/models/sql/compiler.py" in execute_sql
  1334.             for sql, params in self.as_sql():

File "/var/task/django/db/models/sql/compiler.py" in as_sql
  1278.                 for obj in self.query.objs

File "/var/task/django/db/models/sql/compiler.py" in <listcomp>
  1278.                 for obj in self.query.objs

File "/var/task/django/db/models/sql/compiler.py" in <listcomp>
  1277.                 [self.prepare_value(field, self.pre_save_val(field, obj)) for field in fields]

File "/var/task/django/db/models/sql/compiler.py" in pre_save_val
  1228.         return field.pre_save(obj, add=True)

File "/var/task/django/db/models/fields/files.py" in pre_save
  288.             file.save(file.name, file.file, save=False)

File "/var/task/django/db/models/fields/files.py" in save
  87.         self.name = self.storage.save(name, content, max_length=self.field.max_length)

File "/var/task/django/core/files/storage.py" in save
  51.         name = self.get_available_name(name, max_length=max_length)

File "/var/task/storages/backends/s3boto.py" in get_available_name
  525.         return super(S3BotoStorage, self).get_available_name(name, max_length)

File "/var/task/django/core/files/storage.py" in get_available_name
  75.         while self.exists(name) or (max_length and len(name) > max_length):

File "/var/task/storages/backends/s3boto.py" in exists
  464.         return self._get_key(name) is not None

File "/var/task/storages/backends/s3boto.py" in _get_key
  450.         return self.bucket.get_key(self._encode_name(name))

File "/var/task/boto/s3/bucket.py" in get_key
  193.         key, resp = self._get_key_internal(key_name, headers, query_args_l)

File "/var/task/boto/s3/bucket.py" in _get_key_internal
  232.                     response.status, response.reason, '')

Exception Type: S3ResponseError at /myapp/uploadfile
Exception Value: S3ResponseError: 400 Bad Request

I've tried various different configurations of settings and have not come up with a solution. I constantly receive an "S3ResponseError: 400 Bad Request" when I try to upload a file on the Lambda deployment. I have added an S3 endpoint to my VPC, because I assumed it might have something to do with accessing the internet, but I haven't received any time out errors which is what I would expect if that were the issue. I also created a .boto file in my project's main directory (where my zappa_settings.json and manage.py are) with the following:

[Credentials]
aws_access_key_id=***
aws_secret_access_key=****

[s3]
host=s3.us-west-2.amazonaws.com

My settings.py

AWS_S3_HOST = 's3.us-west-2.amazonaws.com'

AWS_S3_ACCESS_KEY_ID = '***'

AWS_S3_SECRET_ACCESS_KEY = '****'

AWS_STORAGE_BUCKET_NAME = 'bucketname'

AWS_QUERYSTRING_AUTH = False

AWS_HEADERS = {'Cache-Control': 'max-age = 86400',}

AWS_STORAGE_REGION = 'us-west-2'

AWS_S3_FILE_OVERWRITE = False

DEFAULT_FILE_STORAGE = "storages.backends.s3boto.S3BotoStorage"

MEDIA_URL = 'http://bucketname.s3.amazonaws.com/'

My file upload form

class CSVUpload(forms.ModelForm):
    class Meta:
        model = UploadedDocument

        fields = ['uploadfile', ]

        labels = {
            'uploadfile': 'Attach CSV or Excel file below (.csv or .xlsx): '
        }

My CreateView for the model

class UploadView(LoginRequiredMixin, PermissionRequiredMixin, CreateView):
    permission_required = 'UploadedDocument.can_add'
    form_class = CSVUpload
    template_name = 'document_upload.html'
    login_url = 'login'
    success_url = 'uploadfile/success'

    def form_valid(self, form):
        form.instance.uploader = self.request.user
        return super(UploadView, self).form_valid(form)

My UploadedDocument model

class UploadedDocument(models.Model):
    """A model representing an uploaded document"""
    id = models.AutoField(primary_key=True)
    uploaddate = models.DateTimeField(auto_now_add=True)
    uploadfile = models.FileField(upload_to='files/', null=True, verbose_name="")
    uploader = models.ForeignKey('users.CustomUser', on_delete=models.PROTECT, null=True)

    def __str__(self):
        return str(self.id)

The HTML page with the form

{% extends 'base.html' %}
{% load crispy_forms_tags %}
{% block title %}CSV upload{% endblock title %}
{% load staticfiles %}

{% block content %}
    <div class="jumbotron">
        <h1>Upload CSV file</h1>
        <form enctype="multipart/form-data" method="post">
            {% csrf_token %}
            {{ form|crispy }}
            <button class="btn btn-success" type="submit">Upload</button>
        </form>
    </div>
{% endblock content %}

Answer 1

Try adding a line to your .boto file:

[s3]
host=s3.us-west-2.amazonaws.com
use-sigv4 = True

If you then get an error telling you When using SigV4, you must specify a 'host' parameter this will help: S3 using boto and SigV4 - missing host parameter