S3 is broken

[im-n1]

Let's assume following django settings where all the env variables are properly set:

STORAGES = {
    "default": {
        "BACKEND": "storages.backends.s3.S3Storage",
        "OPTIONS": {
            "region_name": os.getenv("S3_REGION"),
            "endpoint_url": f"https://{os.getenv('S3_REGION')}.digitaloceanspaces.com",
            "access_key": os.getenv("S3_ACCESS_KEY"),
            "secret_key": os.getenv("S3_SECRET_KEY"),
            "bucket_name": os.getenv("S3_BUCKET"),
            "custom_domain": os.getenv("S3_DOMAIN"),
            "default_acl": "public-read",
            "querystring_auth": False,
            "object_parameters": {
                "CacheControl": "max-age=86400",
            },
        },
    },
}

S3 doesn't work there I do get the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.12/site-packages/django/core/handlers/exception.py", line 55, in inner
    response = get_response(request)
               ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/core/handlers/base.py", line 197, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/views/decorators/csrf.py", line 65, in _view_wrapper
    return view_func(request, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/filer/admin/clipboardadmin.py", line 128, in ajax_upload
    file_obj.save()
  File "/usr/local/lib/python3.12/site-packages/filer/models/imagemodels.py", line 66, in save
    super().save(*args, **kwargs)
  File "/usr/local/lib/python3.12/site-packages/filer/models/abstract.py", line 165, in save
    super().save(*args, **kwargs)
  File "/usr/local/lib/python3.12/site-packages/filer/models/filemodels.py", line 298, in save
    super().save(*args, **kwargs)
  File "/usr/local/lib/python3.12/site-packages/polymorphic/models.py", line 87, in save
    return super().save(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/base.py", line 822, in save
    self.save_base(
  File "/usr/local/lib/python3.12/site-packages/django/db/models/base.py", line 906, in save_base
    parent_inserted = self._save_parents(
                      ^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/base.py", line 959, in _save_parents
    updated = self._save_table(
              ^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/base.py", line 1067, in _save_table
    results = self._do_insert(
              ^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/base.py", line 1108, in _do_insert
    return manager._insert(
           ^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/manager.py", line 87, in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/query.py", line 1847, in _insert
    return query.get_compiler(using=using).execute_sql(returning_fields)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/sql/compiler.py", line 1822, in execute_sql
    for sql, params in self.as_sql():
                       ^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/sql/compiler.py", line 1747, in as_sql
    self.prepare_value(field, self.pre_save_val(field, obj))
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/sql/compiler.py", line 1695, in pre_save_val
    return field.pre_save(obj, add=True)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/db/models/fields/files.py", line 317, in pre_save
    file.save(file.name, file.file, save=False)
  File "/usr/local/lib/python3.12/site-packages/filer/fields/multistorage_file.py", line 120, in save
    super().save(name, content, save)
  File "/usr/local/lib/python3.12/site-packages/easy_thumbnails/files.py", line 643, in save
    super().save(name, content, *args, **kwargs)
  File "/usr/local/lib/python3.12/site-packages/django/db/models/fields/files.py", line 93, in save
    self.name = self.storage.save(name, content, max_length=self.field.max_length)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/django/core/files/storage/base.py", line 38, in save
    name = self._save(name, content)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/storages/backends/s3.py", line 488, in _save
    obj = self.bucket.Object(name)
          ^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/storages/backends/s3.py", line 444, in bucket
    self._bucket = self.connection.Bucket(self.bucket_name)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/boto3/resources/factory.py", line 528, in create_resource
    return partial(
           ^^^^^^^^
  File "/usr/local/lib/python3.12/site-packages/boto3/resources/base.py", line 125, in __init__
    raise ValueError(f'Required parameter {identifier} not set')
ValueError: Required parameter name not set

I tried to put a debug into /boto3/resources/base.py right before the line which raises the exception which is:

# Validate that all identifiers have been set.
for identifier in self.meta.identifiers:
    if getattr(self, identifier) is None:
        raise ValueError(f'Required parameter {identifier} not set')

The function is called many times and self.meta.identifiers has sometimes some sometimes none parameters. But on it's like 4th call the self.meta.identifiers equals ["name"] but the variable is not set under self.itentifier object. One of the previous calls the property is there and corresponds to STORAGES django settings. But not on it's 4th call.

I assume this is not boto3 library error as long as the old approach with django settings AWS_S3_* variables works.

[tonydattolo]

same exact issue. this only occurs for django admin staticfiles. other staticfiles like swagger-ui and debugtoolbar are working

django==4.2.9
django-storages==1.14.2
boto3==1.34.81

STORAGES = {
    "default": {
        "BACKEND": "storages.backends.s3.S3Storage",
        "OPTIONS": {
            "access_key": os.environ["AWS_S3_ACCESS_KEY_ID"],
            "secret_key": os.environ["AWS_S3_SECRET_ACCESS_KEY"],
            "bucket_name": os.environ["AWS_STORAGE_BUCKET_NAME"],
            "region_name": os.environ["AWS_S3_REGION_NAME"],
            "custom_domain": f"{os.environ['AWS_S3_CUSTOM_DOMAIN']}",
            "cloudfront_key": os.environ["AWS_CLOUDFRONT_KEY"].encode("ascii").strip(),
            "cloudfront_key_id": os.environ["AWS_CLOUDFRONT_KEY_ID"].strip(),
            "signature_version": "s3v4",
            "location": "dev/media/",
        },
    },
    "staticfiles": {
        "BACKEND": "storages.backends.s3.S3StaticStorage",
        "OPTIONS": {
            "access_key": os.environ["AWS_S3_ACCESS_KEY_ID"],
            "secret_key": os.environ["AWS_S3_SECRET_ACCESS_KEY"],
            "bucket_name": os.environ["AWS_STORAGE_BUCKET_NAME"],
            "region_name": os.environ["AWS_S3_REGION_NAME"],
            "custom_domain": f"{os.environ['AWS_S3_CUSTOM_DOMAIN_PUBLIC']}",
            "cloudfront_key": os.environ["AWS_CLOUDFRONT_KEY"].encode("ascii").strip(),
            "cloudfront_key_id": os.environ["AWS_CLOUDFRONT_KEY_ID"].strip(),
            "signature_version": "s3v4",
            "location": "prod/static/",
            "querystring_auth": False,
            "default_acl": "public-read",
            "querystring_expire": 604800,
        },
    },
}
STATIC_URL = f"https://{os.environ['AWS_S3_CUSTOM_DOMAIN_PUBLIC']}/prod/static/"
MEDIA_URL = f"https://{os.environ['AWS_S3_CUSTOM_DOMAIN']}/dev/media/"

anyone found a workaround?

[jschneier]

Apologies the fix is in #1336 I simply haven't released it due to life and the like.

[jschneier]

The second comment here where the other staticfiles do not work does seem like some kind of upstream regression. Do you have django.contrib.staticfiles in INSTALLED_APPS?

https://github.com/django/django/blob/main/django/templatetags/static.py#L126-L129