From 0405a67b5e427357cf2b76be4d90e344df886619 Mon Sep 17 00:00:00 2001 From: Wille Marcel Date: Thu, 11 Sep 2025 08:05:35 -0300 Subject: [PATCH] Use django-storages to upload raster file to S3-like bucket + set names as unique --- .env.example | 4 ++ README.md | 6 ++- docker-compose.yml | 7 ++- requirements.txt | 5 ++ vbos/config/production.py | 16 ++++++- ...set_name_alter_rasterfile_file_and_more.py | 47 +++++++++++++++++++ vbos/datasets/models.py | 14 ++++-- vbos/datasets/test/test_raster_models.py | 17 ++++++- 8 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 .env.example create mode 100644 vbos/datasets/migrations/0004_alter_rasterdataset_name_alter_rasterfile_file_and_more.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3b57db0 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +DJANGO_SECRET_KEY="local" +DJANGO_AWS_ACCESS_KEY_ID="" +DJANGO_AWS_SECRET_ACCESS_KEY="" +DJANGO_AWS_STORAGE_BUCKET_NAME="" diff --git a/README.md b/README.md index 7f17a67..d8728cc 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ VBOS Django application and data services. Check out the project's [documentatio # Prerequisites -- [Docker](https://docs.docker.com/docker-for-mac/install/) +- [Docker](https://docs.docker.com/docker-for-mac/install/) # Local Development @@ -21,3 +21,7 @@ Run a command inside the docker container: ```bash docker-compose run --rm web [command] ``` + +# Configuration + +Copy the `.env.example` file to `.env` and edit the variables you need to configure the access to the DigitalOcean Spaces (S3 compatible). diff --git a/docker-compose.yml b/docker-compose.yml index 613d0ac..843a0b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,10 @@ services: web: restart: always environment: - - DJANGO_SECRET_KEY=local + - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} + - DJANGO_AWS_ACCESS_KEY_ID=${DJANGO_AWS_ACCESS_KEY_ID} + - DJANGO_AWS_SECRET_ACCESS_KEY=${DJANGO_AWS_SECRET_ACCESS_KEY} + - DJANGO_AWS_STORAGE_BUCKET_NAME=${DJANGO_AWS_STORAGE_BUCKET_NAME} build: ./ command: > bash -c "python3 wait_for_postgres.py && @@ -54,5 +57,5 @@ services: ports: - "8002:8000" volumes: - - ./data:/data # Optional: mount local directory with your raster files + - ./data:/data # Optional: mount local directory with your raster files restart: unless-stopped diff --git a/requirements.txt b/requirements.txt index 7af577b..1c5bae5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,11 @@ drf_spectacular==0.28.0 django-cors-headers==4.7.0 drf-excel==2.5.3 +# Storage +django-storages==1.14.6 +boto3==1.40.26 +whitenoise + # Developer Tools ipdb==0.13.13 ipython==8.30.0 diff --git a/vbos/config/production.py b/vbos/config/production.py index d263c44..9f943b2 100755 --- a/vbos/config/production.py +++ b/vbos/config/production.py @@ -3,6 +3,7 @@ class Production(Common): + DEBUG = False INSTALLED_APPS = Common.INSTALLED_APPS SECRET_KEY = os.getenv("DJANGO_SECRET_KEY") # Site @@ -13,13 +14,26 @@ class Production(Common): # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ # http://django-storages.readthedocs.org/en/latest/index.html + INSTALLED_APPS += ("storages",) + STORAGES = { + "default": {"BACKEND": "storages.backends.s3.S3Storage"}, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage" + }, + } + AWS_S3_OBJECT_PARAMETERS = { + "CacheControl": "max-age=2592000", + } + AWS_S3_ENDPOINT_URL = os.getenv( + "DJANGO_AWS_S3_ENDPOINT_URL", "https://syd1.digitaloceanspaces.com" + ) AWS_ACCESS_KEY_ID = os.getenv("DJANGO_AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = os.getenv("DJANGO_AWS_SECRET_ACCESS_KEY") AWS_STORAGE_BUCKET_NAME = os.getenv("DJANGO_AWS_STORAGE_BUCKET_NAME") AWS_DEFAULT_ACL = "public-read" AWS_AUTO_CREATE_BUCKET = True AWS_QUERYSTRING_AUTH = False - MEDIA_URL = f"https://s3.amazonaws.com/{AWS_STORAGE_BUCKET_NAME}/" + MEDIA_URL = f"https://{AWS_STORAGE_BUCKET_NAME}.syd1.digitaloceanspaces.com/" # https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/http-caching#cache-control # Response can be cached by browser and any intermediary caches (i.e. it is "public") for up to 1 day diff --git a/vbos/datasets/migrations/0004_alter_rasterdataset_name_alter_rasterfile_file_and_more.py b/vbos/datasets/migrations/0004_alter_rasterdataset_name_alter_rasterfile_file_and_more.py new file mode 100644 index 0000000..131724f --- /dev/null +++ b/vbos/datasets/migrations/0004_alter_rasterdataset_name_alter_rasterfile_file_and_more.py @@ -0,0 +1,47 @@ +# Generated by Django 5.2.5 on 2025-09-11 11:04 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("datasets", "0003_rasterfile_rasterdataset"), + ] + + operations = [ + migrations.AlterField( + model_name="rasterdataset", + name="name", + field=models.CharField(max_length=155, unique=True), + ), + migrations.AlterField( + model_name="rasterfile", + name="file", + field=models.FileField( + unique=True, + upload_to="staging/raster/", + validators=[ + django.core.validators.FileExtensionValidator( + allowed_extensions=["tiff", "tif", "geotiff", "gtiff"] + ) + ], + ), + ), + migrations.AlterField( + model_name="rasterfile", + name="name", + field=models.CharField(max_length=155, unique=True), + ), + migrations.AlterField( + model_name="tabulardataset", + name="name", + field=models.CharField(max_length=155, unique=True), + ), + migrations.AlterField( + model_name="vectordataset", + name="name", + field=models.CharField(max_length=155, unique=True), + ), + ] diff --git a/vbos/datasets/models.py b/vbos/datasets/models.py index 0c50b16..63228e9 100644 --- a/vbos/datasets/models.py +++ b/vbos/datasets/models.py @@ -1,15 +1,19 @@ from django.contrib.gis.db import models +from django.conf import settings from django.core.validators import FileExtensionValidator from django.db.models.fields.files import default_storage from django.db.models.signals import pre_delete from django.dispatch import receiver +UPLOAD_TO = "staging/raster/" if settings.DEBUG else "production/raster/" + class RasterFile(models.Model): - name = models.CharField(max_length=155) + name = models.CharField(max_length=155, unique=True) created = models.DateTimeField(auto_now_add=True) file = models.FileField( - upload_to="raster/", + upload_to=UPLOAD_TO, + unique=True, validators=[ FileExtensionValidator( allowed_extensions=["tiff", "tif", "geotiff", "gtiff"] @@ -36,7 +40,7 @@ def delete_raster_file(sender, instance, **kwargs): class RasterDataset(models.Model): - name = models.CharField(max_length=155) + name = models.CharField(max_length=155, unique=True) created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) file = models.ForeignKey(RasterFile, on_delete=models.PROTECT) @@ -49,7 +53,7 @@ class Meta: class VectorDataset(models.Model): - name = models.CharField(max_length=155) + name = models.CharField(max_length=155, unique=True) created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) @@ -73,7 +77,7 @@ class Meta: class TabularDataset(models.Model): - name = models.CharField(max_length=155) + name = models.CharField(max_length=155, unique=True) created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) diff --git a/vbos/datasets/test/test_raster_models.py b/vbos/datasets/test/test_raster_models.py index 36dbf69..5d158e5 100644 --- a/vbos/datasets/test/test_raster_models.py +++ b/vbos/datasets/test/test_raster_models.py @@ -4,13 +4,12 @@ from django.core.exceptions import ValidationError from vbos.datasets.models import RasterDataset, RasterFile -from genericpath import exists class TestRasterModels(TestCase): def setUp(self): self.valid_file = SimpleUploadedFile( - "rainfall.tif", b"file_content", content_type="image/tiff" + "rainfall.tiff", b"file_content", content_type="image/tiff" ) self.r_1 = RasterFile.objects.create(name="Rainfall COG", file=self.valid_file) self.r_2 = RasterFile.objects.create( @@ -22,6 +21,17 @@ def test_deletion(self): # RasterFile can't be deleted if it's associates with a dataset with self.assertRaises(ProtectedError): self.r_1.delete() + + # name should be unique + raster = RasterFile(name="Rainfall COG 2", file="raster/coastline.tiff") + with self.assertRaises(ValidationError): + raster.full_clean() + + # file path should be unique + raster = RasterFile(name="Rainfall COG", file="newfile.tif") + with self.assertRaises(ValidationError): + raster.full_clean() + # modify dataset self.dataset.file = self.r_2 self.dataset.save() @@ -42,3 +52,6 @@ def test_deletion(self): raster = RasterFile(name="Test", file=invalid_file) with self.assertRaises(ValidationError): raster.full_clean() + + def tearDown(self): + RasterFile.objects.all().delete()