From ae8c606d8454c35da3efa77bfcdf6d6d86f1834a Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Thu, 14 May 2026 10:43:40 -0500 Subject: [PATCH 01/14] add xyz tiles output to existing mosaic command --- ohmg/core/admin.py | 1 + .../0012_layerset_xyz_tiles_prefix.py | 18 ++++ ohmg/core/models/layerset.py | 16 ++++ ohmg/core/utils/__init__.py | 12 +++ .../management/commands/mosaic.py | 11 ++- ohmg/georeference/models.py | 8 +- ohmg/georeference/mosaicker.py | 88 +++++++++++++++++-- 7 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 ohmg/core/migrations/0012_layerset_xyz_tiles_prefix.py diff --git a/ohmg/core/admin.py b/ohmg/core/admin.py index 96660f5b..5ec5a4a5 100644 --- a/ohmg/core/admin.py +++ b/ohmg/core/admin.py @@ -66,6 +66,7 @@ class LayerSetAdmin(admin.ModelAdmin): "layer_display_list", "extent", "multimask_extent", + "xyz_tiles_url", ) search_fields = ("map__title",) list_filter = ("category",) diff --git a/ohmg/core/migrations/0012_layerset_xyz_tiles_prefix.py b/ohmg/core/migrations/0012_layerset_xyz_tiles_prefix.py new file mode 100644 index 00000000..d2bcae6a --- /dev/null +++ b/ohmg/core/migrations/0012_layerset_xyz_tiles_prefix.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.27 on 2026-05-13 10:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0011_map_main_layer_ct'), + ] + + operations = [ + migrations.AddField( + model_name='layerset', + name='xyz_tiles_prefix', + field=models.CharField(blank=True, max_length=200, null=True), + ), + ] diff --git a/ohmg/core/models/layerset.py b/ohmg/core/models/layerset.py index 3401f768..a61ecb5b 100644 --- a/ohmg/core/models/layerset.py +++ b/ohmg/core/models/layerset.py @@ -64,6 +64,11 @@ class Meta: null=True, blank=True, ) + xyz_tiles_prefix = models.CharField( + max_length=200, + blank=True, + null=True, + ) tilejson = models.JSONField(null=True, blank=True) def __str__(self): @@ -81,6 +86,17 @@ def layer_display_list(self): def get_layers(self) -> Iterable["Layer"]: return self.layer_set.all() + @property + def xyz_tiles_url(self): + if self.xyz_tiles_prefix: + if settings.ENABLE_S3_STORAGE: + base_url = f"{settings.AWS_S3_ENDPOINT_URL}/{settings.AWS_STORAGE_BUCKET_NAME}" + else: + base_url = f"{settings.SITEURL.rstrip('/')}{settings.MEDIA_URL}" + return f"{base_url.rstrip('/')}/{self.xyz_tiles_prefix}" + else: + return None + @cached_property def centroid(self): return Polygon.from_bbox(self.extent).centroid diff --git a/ohmg/core/utils/__init__.py b/ohmg/core/utils/__init__.py index f78cea99..2ed79efb 100644 --- a/ohmg/core/utils/__init__.py +++ b/ohmg/core/utils/__init__.py @@ -36,6 +36,18 @@ def random_alnum(size=6): return code +def get_boto3_s3_client(): + import boto3 + + return boto3.client( + "s3", + region_name=settings.AWS_S3_REGION_NAME, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + endpoint_url=settings.AWS_S3_ENDPOINT_URL, + ) + + MONTH_CHOICES = [ (1, "JAN."), (2, "FEB."), diff --git a/ohmg/georeference/management/commands/mosaic.py b/ohmg/georeference/management/commands/mosaic.py index 3da4773c..235de010 100644 --- a/ohmg/georeference/management/commands/mosaic.py +++ b/ohmg/georeference/management/commands/mosaic.py @@ -15,6 +15,7 @@ def add_arguments(self, parser): "operation", choices=[ "generate-cog", + "generate-tiles", ], help="the operation to perform", ) @@ -48,10 +49,18 @@ def handle(self, *args, **options): map__identifier=options.mapid, category__slug=options.category ) + m = Mosaicker() + + if options.operation == "generate-tiles": + if options.background: + create_mosaic_cog.delay(ls.pk) + else: + m.generate_xyz_tiles(ls) + m.cleanup_files() + if options.operation == "generate-cog": if options.background: create_mosaic_cog.delay(ls.pk) else: - m = Mosaicker() m.generate_cog(ls) m.cleanup_files() diff --git a/ohmg/georeference/models.py b/ohmg/georeference/models.py index d15580ef..6b84b34b 100644 --- a/ohmg/georeference/models.py +++ b/ohmg/georeference/models.py @@ -711,7 +711,7 @@ def run(self): ## regardless of whether there was an old layer or not, overwrite ## the file with the newly georeferenced tif. session_ct = GeorefSession.objects.filter(reg2=self.reg2).exclude(pk=self.pk).count() - file_name = f"{layer.slug}__{random_alnum(6)}_{str(session_ct).zfill(2)}.tif" + file_name = f"{layer.slug}__{random_alnum()}_{str(session_ct).zfill(2)}.tif" with open(g.cog, "rb") as openf: layer.file.save(file_name, File(openf)) @@ -793,8 +793,10 @@ class Meta: def __str__(self): return ( - f"{self.session} --> {self.target._meta.object_name} ({self.target} {self.target_id})" - ) if self.target else f"{self.session} --> (no target)" + (f"{self.session} --> {self.target._meta.object_name} ({self.target} {self.target_id})") + if self.target + else f"{self.session} --> (no target)" + ) def extend(self): self.expiration += timedelta(seconds=settings.GEOREFERENCE_SESSION_LENGTH) diff --git a/ohmg/georeference/mosaicker.py b/ohmg/georeference/mosaicker.py index eea0d3bf..718d2023 100644 --- a/ohmg/georeference/mosaicker.py +++ b/ohmg/georeference/mosaicker.py @@ -1,6 +1,8 @@ import json import logging import os +import shutil +import subprocess from datetime import datetime from glob import glob from pathlib import Path @@ -14,7 +16,7 @@ from ohmg.core.models import Layer, LayerSet from ohmg.core.storages import get_file_url -from ohmg.core.utils import random_alnum +from ohmg.core.utils import get_boto3_s3_client, random_alnum from .georeferencer import Georeferencer, VRTHandler @@ -62,9 +64,11 @@ def generate_mosaic_vrt(self, layerset) -> VRTHandler: print(layer_name) try: layer = Layer.objects.get(slug=layer_name, region__document__map=layerset.map) - except Layer.MultipleObjectsReturned as e: - print("this layer slug matched multiple layers in this map: cancelling mosaic process") - except Exception as e: + except Layer.MultipleObjectsReturned: + print( + "this layer slug matched multiple layers in this map: cancelling mosaic process" + ) + except Exception as e: raise e if not layer.file: @@ -124,7 +128,7 @@ def generate_cog(self, layerset: LayerSet): existing_file_name = layerset.mosaic_geotiff.name if layerset.mosaic_geotiff else None - file_name = f"{layerset.map.identifier}-{layerset.category.slug}__{datetime.now().strftime('%Y-%m-%d')}__{random_alnum(6)}.tif" + file_name = f"{layerset.map.identifier}-{layerset.category.slug}__{datetime.now().strftime('%Y-%m-%d')}__{random_alnum()}.tif" with open(self.cog, "rb") as f: layerset.mosaic_geotiff.save(file_name, File(f)) @@ -137,6 +141,78 @@ def generate_cog(self, layerset: LayerSet): print(f"completed - elapsed time: {datetime.now() - start}") + def generate_xyz_tiles(self, layerset: LayerSet): + start = datetime.now() + + self.generate_mosaic_vrt(layerset) + + prefix = f"tiles/{layerset.map.identifier}/{layerset.category.slug}/{random_alnum()}" + logger.info(f"creating new tileset {prefix}") + if settings.ENABLE_S3_STORAGE: + out_path = Path(settings.TEMP_DIR, prefix) + else: + out_path = Path(settings.MEDIA_ROOT, prefix) + + cmd = [ + "gdal2tiles.py", + "--xyz", + "-z", + "13-20", + str(self.mosaic_vrt.get_path()), + out_path, + ] + + try: + subprocess.run(cmd, check=True) + logger.info(f"{prefix} tileset created, elapsed time: {datetime.now() - start}") + except subprocess.CalledProcessError as e: + raise Exception(f"Error during tile generation: {e}") + + if settings.ENABLE_S3_STORAGE: + s3 = get_boto3_s3_client() + + all_files = [] + for root, dirs, files in os.walk(out_path): + for f in files: + all_files.append( + {"path": Path(root, f), "key": Path(root, f).relative_to(settings.TEMP_DIR)} + ) + + logger.debug( + f"uploading {len(all_files)} tiles to bucket: {settings.AWS_STORAGE_BUCKET_NAME}" + ) + for f in all_files: + s3.upload_file( + f["path"], + settings.AWS_STORAGE_BUCKET_NAME, + str(f["key"]), + ExtraArgs={"ACL": "public-read"}, + ) + + logger.debug("deleting temp local tileset") + shutil.rmtree(out_path) + + existing_tileset_prefix = layerset.xyz_tiles_prefix + layerset.xyz_tiles_prefix = prefix + layerset.save() + + ## clean up existing tileset + if existing_tileset_prefix: + logger.info(f"deleting existing tileset {existing_tileset_prefix}") + if settings.ENABLE_S3_STORAGE: + s3 = get_boto3_s3_client() + response = s3.list_objects_v2( + Bucket=settings.AWS_STORAGE_BUCKET_NAME, Prefix=existing_tileset_prefix + ) + logger.info(f"deleting {len(response['Contents'])} tiles") + for object in response["Contents"]: + s3.delete_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=object["Key"]) + else: + shutil.rmtree( + Path(settings.MEDIA_ROOT, existing_tileset_prefix), + ignore_errors=True, + ) + def generate_mosaic_json(self, layerset, trim_all=False): """DEPRECATED: Currently, MosaicJSON is not used anywhere in the app.""" from cogeo_mosaic.backends import MosaicBackend @@ -188,7 +264,7 @@ def read_trim_feature_cache(file_path): cached_feature = None write_trim_feature_cache(feature, feat_cache_path) - unique_id = random_alnum(6) + unique_id = random_alnum() trim_vrt_path = in_path.replace(".tif", f"_{unique_id}_trim.vrt") out_path = trim_vrt_path.replace(".vrt", ".tif") From ec3c4c7b2ba7599fda858738e489688ad90b3b15 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Thu, 14 May 2026 13:30:55 -0500 Subject: [PATCH 02/14] allow tileset command to run in background --- ohmg/conf/settings.py | 1 + ohmg/georeference/management/commands/mosaic.py | 4 ++-- ohmg/georeference/tasks.py | 13 ++++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ohmg/conf/settings.py b/ohmg/conf/settings.py index 83331290..b3c425aa 100644 --- a/ohmg/conf/settings.py +++ b/ohmg/conf/settings.py @@ -323,6 +323,7 @@ "ohmg.georeference.tasks.delete_stale_sessions": {"queue": "housekeeping"}, "ohmg.georeference.tasks.delete_preview_vrts": {"queue": "housekeeping"}, "ohmg.georeference.tasks.create_mosaic_cog": {"queue": "mosaic"}, + "ohmg.georeference.tasks.create_mosaic_tileset": {"queue": "mosaic"}, "ohmg.core.tasks.load_map_documents_as_task": {"queue": "map"}, "ohmg.core.tasks.load_document_file_as_task": {"queue": "map"}, } diff --git a/ohmg/georeference/management/commands/mosaic.py b/ohmg/georeference/management/commands/mosaic.py index 235de010..81fd0861 100644 --- a/ohmg/georeference/management/commands/mosaic.py +++ b/ohmg/georeference/management/commands/mosaic.py @@ -4,7 +4,7 @@ from ohmg.core.models import LayerSet from ohmg.georeference.mosaicker import Mosaicker -from ohmg.georeference.tasks import create_mosaic_cog +from ohmg.georeference.tasks import create_mosaic_cog, create_mosaic_tileset class Command(BaseCommand): @@ -53,7 +53,7 @@ def handle(self, *args, **options): if options.operation == "generate-tiles": if options.background: - create_mosaic_cog.delay(ls.pk) + create_mosaic_tileset.delay(ls.pk) else: m.generate_xyz_tiles(ls) m.cleanup_files() diff --git a/ohmg/georeference/tasks.py b/ohmg/georeference/tasks.py index 7fc5e4ca..8adea8fb 100644 --- a/ohmg/georeference/tasks.py +++ b/ohmg/georeference/tasks.py @@ -39,7 +39,7 @@ def run_georeference_session(sessionid): return session.pk -@app.task() +@app.task def delete_stale_sessions(): delete_expired_session_locks() @@ -59,3 +59,14 @@ def create_mosaic_cog(layersetid): m = Mosaicker() m.generate_cog(layerset) m.cleanup_files() + + +@app.task +def create_mosaic_tileset(layersetid): + try: + layerset = LayerSet.objects.get(pk=layersetid) + except LayerSet.DoesNotExist: + logger.warning(f"LayerSet does not exist: {layersetid}. Cancelling mosaic creation.") + m = Mosaicker() + m.generate_xyz_tiles(layerset) + m.cleanup_files() From 9af0cd02d9ca9a42bd7236290fff174b42d38226 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Thu, 14 May 2026 13:31:36 -0500 Subject: [PATCH 03/14] split mosaic queue into new celery worker --- .../management/commands/configure-services.py | 57 +++++++++++++++---- scripts/celery_dev.sh | 3 - scripts/celery_main_dev.sh | 12 ++++ scripts/celery_mosaic_dev.sh | 12 ++++ 4 files changed, 70 insertions(+), 14 deletions(-) delete mode 100644 scripts/celery_dev.sh create mode 100644 scripts/celery_main_dev.sh create mode 100644 scripts/celery_mosaic_dev.sh diff --git a/ohmg/conf/management/commands/configure-services.py b/ohmg/conf/management/commands/configure-services.py index 420bca47..d57c758a 100644 --- a/ohmg/conf/management/commands/configure-services.py +++ b/ohmg/conf/management/commands/configure-services.py @@ -31,8 +31,11 @@ def handle(self, *args, **options): out_dir = Path(options["destination"]) out_dir.mkdir(exist_ok=True) - cs_path = Path(out_dir, "celery.service") - self._write_file(self.generate_celery_service(out_dir), cs_path) + cs1_path = Path(out_dir, "celery_main.service") + self._write_file(self.generate_celery_main_service(out_dir), cs1_path) + + cs2_path = Path(out_dir, "celery_mosaic.service") + self._write_file(self.generate_celery_mosaic_service(out_dir), cs2_path) ui_path = Path(out_dir, "uwsgi.ini") self._write_file(self.generate_uwsgi_ini(), ui_path) @@ -43,21 +46,25 @@ def handle(self, *args, **options): print(f"""services created. to deploy, run the following commands: # initial deployment (first time only) -sudo ln -sf {cs_path.absolute()} /etc/systemd/system +sudo ln -sf {cs1_path.absolute()} /etc/systemd/system +sudo ln -sf {cs2_path.absolute()} /etc/systemd/system sudo ln -sf {us_path.absolute()} /etc/systemd/system sudo systemctl daemon-reload -sudo systemctl enable celery +sudo systemctl enable celery_main +sudo systemctl enable celery_mosaic sudo systemctl enable uwsgi -sudo systemctl start celery +sudo systemctl start celery_main +sudo systemctl start celery_mosaic sudo systemctl start uwsgi -# reload services +# reload services after changes sudo systemctl daemon-reload -sudo systemctl restart celery +sudo systemctl restart celery_main +sudo systemctl restart celery_mosaic sudo systemctl restart uwsgi """) - output_files = [cs_path, ui_path, us_path] + output_files = [cs1_path, cs2_path, ui_path, us_path] if self.verbose: print(f"~~~\noutput directory: {out_dir.absolute()}") @@ -167,7 +174,7 @@ def generate_uwsgi_service(self, ini_file_path: Path): """ return file_content - def generate_celery_service(self, state_path: Path): + def generate_celery_main_service(self, state_path: Path): log_dir = self._resolve_var("LOG_DIR", settings.LOG_DIR) file_content = f"""[Unit] @@ -179,15 +186,43 @@ def generate_celery_service(self, state_path: Path): EnvironmentFile={settings.BASE_DIR}/.env ExecStart={self.python_env}/celery \\ -A ohmg.conf.celery:app worker \\ + -Q split,georeference,map,housekeeping \\ --without-gossip --without-mingle \\ -Ofair -B -E \\ - --statedb={str(state_path.resolve())}/worker.state \\ + --statedb={str(state_path.resolve())}/celery_main_worker.state \\ --schedule-filename={str(state_path.resolve())}/celerybeat-schedule \\ --loglevel=INFO \\ - --logfile={log_dir}/celery.log \\ + --logfile={log_dir}/celery_main.log \\ --concurrency=10 -n worker1@%h Restart=always +[Install] +WantedBy=multi-user.target +""" + + return file_content + + def generate_celery_mosaic_service(self, state_path: Path): + log_dir = self._resolve_var("LOG_DIR", settings.LOG_DIR) + + file_content = f"""[Unit] +Description=Celery +After=rabbitmq-server.service +Requires=rabbitmq-server.service + +[Service] +EnvironmentFile={settings.BASE_DIR}/.env +ExecStart={self.python_env}/celery \\ + -A ohmg.conf.celery:app worker \\ + -Q mosaic \\ + --without-gossip --without-mingle \\ + -Ofair -B -E \\ + --statedb={str(state_path.resolve())}/celery_mosaic_worker.state \\ + --loglevel=INFO \\ + --logfile={log_dir}/celery_mosaic.log \\ + --concurrency=1 -n worker2@%h +Restart=always + [Install] WantedBy=multi-user.target """ diff --git a/scripts/celery_dev.sh b/scripts/celery_dev.sh deleted file mode 100644 index 45e703dc..00000000 --- a/scripts/celery_dev.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /usr/bin/bash - -celery -A ohmg.conf.celery:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=DEBUG --concurrency=10 -n worker1@%h diff --git a/scripts/celery_main_dev.sh b/scripts/celery_main_dev.sh new file mode 100644 index 00000000..5b390881 --- /dev/null +++ b/scripts/celery_main_dev.sh @@ -0,0 +1,12 @@ +#! /usr/bin/bash + +uv run celery -A ohmg.conf.celery:app worker \ + -Q split,georeference,map,housekeeping \ + --without-gossip \ + --without-mingle \ + -Ofair -B -E \ + --statedb=worker.state \ + -s celerybeat-schedule \ + --loglevel=DEBUG \ + --concurrency=10 \ + -n worker1@%h diff --git a/scripts/celery_mosaic_dev.sh b/scripts/celery_mosaic_dev.sh new file mode 100644 index 00000000..35ccae1a --- /dev/null +++ b/scripts/celery_mosaic_dev.sh @@ -0,0 +1,12 @@ +#! /usr/bin/bash + +uv run celery -A ohmg.conf.celery:app worker \ + -Q mosaic \ + --without-gossip \ + --without-mingle \ + -Ofair -B -E \ + --statedb=worker.state \ + -s celerybeat-schedule \ + --loglevel=DEBUG \ + --concurrency=1 \ + -n worker2@%h From efc54682d8ef4ee35d840edde6a639c2e31b79db Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Thu, 14 May 2026 16:06:03 -0500 Subject: [PATCH 04/14] consolidate task queues --- ohmg/conf/settings.py | 26 +++++++++++++------------- scripts/celery_main_dev.sh | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ohmg/conf/settings.py b/ohmg/conf/settings.py index b3c425aa..e3d9e629 100644 --- a/ohmg/conf/settings.py +++ b/ohmg/conf/settings.py @@ -307,25 +307,25 @@ # basic independent setup for Celery Exchange/Queue DEFAULT_EXCHANGE = Exchange("default", type="topic") -# CELERY_TASK_QUEUES += ( + CELERY_TASK_QUEUES = ( - Queue("split", DEFAULT_EXCHANGE, routing_key="split", priority=0), - Queue("georeference", DEFAULT_EXCHANGE, routing_key="georeference", priority=0), - Queue("map", DEFAULT_EXCHANGE, routing_key="map", priority=0), - Queue("mosaic", DEFAULT_EXCHANGE, routing_key="mosaic", priority=0), - Queue("housekeeping", DEFAULT_EXCHANGE, routing_key="housekeeping", priority=0), + ## these queues run on the same worker + Queue("main", DEFAULT_EXCHANGE, routing_key="main"), + Queue("background", DEFAULT_EXCHANGE, routing_key="background"), + ## this queue runs on its own worker + Queue("mosaic", DEFAULT_EXCHANGE, routing_key="mosaic"), ) CELERY_TASK_ROUTES = { - "ohmg.georeference.tasks.run_preparation_session": {"queue": "split"}, - "ohmg.georeference.tasks.bulk_run_preparation_sessions": {"queue": "split"}, - "ohmg.georeference.tasks.run_georeference_session": {"queue": "georeference"}, - "ohmg.georeference.tasks.delete_stale_sessions": {"queue": "housekeeping"}, - "ohmg.georeference.tasks.delete_preview_vrts": {"queue": "housekeeping"}, + "ohmg.georeference.tasks.run_preparation_session": {"queue": "main"}, + "ohmg.georeference.tasks.bulk_run_preparation_sessions": {"queue": "main"}, + "ohmg.georeference.tasks.run_georeference_session": {"queue": "main"}, + "ohmg.core.tasks.load_map_documents_as_task": {"queue": "main"}, + "ohmg.core.tasks.load_document_file_as_task": {"queue": "main"}, + "ohmg.georeference.tasks.delete_stale_sessions": {"queue": "background"}, + "ohmg.georeference.tasks.delete_preview_vrts": {"queue": "background"}, "ohmg.georeference.tasks.create_mosaic_cog": {"queue": "mosaic"}, "ohmg.georeference.tasks.create_mosaic_tileset": {"queue": "mosaic"}, - "ohmg.core.tasks.load_map_documents_as_task": {"queue": "map"}, - "ohmg.core.tasks.load_document_file_as_task": {"queue": "map"}, } # empty celery beat schedule of default GeoNode jobs diff --git a/scripts/celery_main_dev.sh b/scripts/celery_main_dev.sh index 5b390881..e92839fe 100644 --- a/scripts/celery_main_dev.sh +++ b/scripts/celery_main_dev.sh @@ -1,7 +1,7 @@ #! /usr/bin/bash uv run celery -A ohmg.conf.celery:app worker \ - -Q split,georeference,map,housekeeping \ + -Q main,background \ --without-gossip \ --without-mingle \ -Ofair -B -E \ From 0d011d48697b2b7653569835c79e65d952d25ad9 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sat, 16 May 2026 16:55:41 +0000 Subject: [PATCH 05/14] update queues in service files --- ohmg/conf/management/commands/configure-services.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ohmg/conf/management/commands/configure-services.py b/ohmg/conf/management/commands/configure-services.py index d57c758a..270da862 100644 --- a/ohmg/conf/management/commands/configure-services.py +++ b/ohmg/conf/management/commands/configure-services.py @@ -186,7 +186,7 @@ def generate_celery_main_service(self, state_path: Path): EnvironmentFile={settings.BASE_DIR}/.env ExecStart={self.python_env}/celery \\ -A ohmg.conf.celery:app worker \\ - -Q split,georeference,map,housekeeping \\ + -Q main,background \\ --without-gossip --without-mingle \\ -Ofair -B -E \\ --statedb={str(state_path.resolve())}/celery_main_worker.state \\ From 0196c5fbaeca5eb9514844a8e8444f017fd716d8 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sat, 16 May 2026 14:49:35 -0500 Subject: [PATCH 06/14] delete existing tileset as task --- ohmg/conf/settings.py | 1 + ohmg/georeference/tasks.py | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/ohmg/conf/settings.py b/ohmg/conf/settings.py index e3d9e629..9748d8e9 100644 --- a/ohmg/conf/settings.py +++ b/ohmg/conf/settings.py @@ -324,6 +324,7 @@ "ohmg.core.tasks.load_document_file_as_task": {"queue": "main"}, "ohmg.georeference.tasks.delete_stale_sessions": {"queue": "background"}, "ohmg.georeference.tasks.delete_preview_vrts": {"queue": "background"}, + "ohmg.georeference.tasks.cleanup_existing_tileset": {"queue": "background"}, "ohmg.georeference.tasks.create_mosaic_cog": {"queue": "mosaic"}, "ohmg.georeference.tasks.create_mosaic_tileset": {"queue": "mosaic"}, } diff --git a/ohmg/georeference/tasks.py b/ohmg/georeference/tasks.py index 8adea8fb..99155a35 100644 --- a/ohmg/georeference/tasks.py +++ b/ohmg/georeference/tasks.py @@ -1,12 +1,13 @@ import logging import os +import shutil from pathlib import Path from django.conf import settings from ohmg.conf.celery import app from ohmg.core.models import LayerSet -from ohmg.georeference.mosaicker import Mosaicker +from ohmg.core.utils import get_boto3_s3_client from .models import ( GeorefSession, @@ -52,6 +53,8 @@ def delete_preview_vrts(id): @app.task def create_mosaic_cog(layersetid): + from ohmg.georeference.mosaicker import Mosaicker + try: layerset = LayerSet.objects.get(pk=layersetid) except LayerSet.DoesNotExist: @@ -63,6 +66,8 @@ def create_mosaic_cog(layersetid): @app.task def create_mosaic_tileset(layersetid): + from ohmg.georeference.mosaicker import Mosaicker + try: layerset = LayerSet.objects.get(pk=layersetid) except LayerSet.DoesNotExist: @@ -70,3 +75,19 @@ def create_mosaic_tileset(layersetid): m = Mosaicker() m.generate_xyz_tiles(layerset) m.cleanup_files() + + +@app.task +def cleanup_existing_tileset(prefix): + logger.info(f"deleting existing tileset {prefix}") + if settings.ENABLE_S3_STORAGE: + s3 = get_boto3_s3_client() + response = s3.list_objects_v2(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Prefix=prefix) + logger.info(f"deleting {len(response['Contents'])} tiles") + for object in response["Contents"]: + s3.delete_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=object["Key"]) + else: + shutil.rmtree( + Path(settings.MEDIA_ROOT, prefix), + ignore_errors=True, + ) From 5a28eb24ec0827a8855fcb9f3795a65113924437 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sat, 16 May 2026 14:55:49 -0500 Subject: [PATCH 07/14] generate xyz tiles with riotiler instead of gdal2tiles --- ohmg/georeference/mosaicker.py | 47 +++++++++++++--------------------- pyproject.toml | 3 ++- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/ohmg/georeference/mosaicker.py b/ohmg/georeference/mosaicker.py index 718d2023..d8a739a4 100644 --- a/ohmg/georeference/mosaicker.py +++ b/ohmg/georeference/mosaicker.py @@ -2,23 +2,25 @@ import logging import os import shutil -import subprocess from datetime import datetime from glob import glob from pathlib import Path from typing import List +import morecantile from django.conf import settings from django.contrib.gis.geos import MultiPolygon, Polygon from django.core.files import File from django.core.files.storage import get_storage_class from osgeo import gdal +from rio_tiler.io import Reader from ohmg.core.models import Layer, LayerSet from ohmg.core.storages import get_file_url from ohmg.core.utils import get_boto3_s3_client, random_alnum from .georeferencer import Georeferencer, VRTHandler +from .tasks import cleanup_existing_tileset gdal.SetConfigOption("GDAL_NUM_THREADS", "ALL_CPUS") gdal.SetConfigOption("GDAL_TIFF_INTERNAL_MASK", "YES") @@ -141,10 +143,11 @@ def generate_cog(self, layerset: LayerSet): print(f"completed - elapsed time: {datetime.now() - start}") - def generate_xyz_tiles(self, layerset: LayerSet): + def generate_xyz_tiles(self, layerset: LayerSet, min_zoom: int = 13, max_zoom: int = 20): start = datetime.now() self.generate_mosaic_vrt(layerset) + tms = morecantile.tms.get("WebMercatorQuad") prefix = f"tiles/{layerset.map.identifier}/{layerset.category.slug}/{random_alnum()}" logger.info(f"creating new tileset {prefix}") @@ -153,20 +156,19 @@ def generate_xyz_tiles(self, layerset: LayerSet): else: out_path = Path(settings.MEDIA_ROOT, prefix) - cmd = [ - "gdal2tiles.py", - "--xyz", - "-z", - "13-20", - str(self.mosaic_vrt.get_path()), - out_path, - ] + with Reader(self.mosaic_vrt.get_path()) as src: + for coords in tms.tiles(*src.geographic_bounds, zooms=range(min_zoom, max_zoom + 1)): + tile = src.tile(coords.x, coords.y, coords.z) + ## only make a tile if there is valid data (skip empty tiles) + if tile.data_as_image().any(): + rendered_bytes = tile.render() + out_dir = Path(out_path, str(coords.z), str(coords.x)) + out_dir.mkdir(parents=True, exist_ok=True) + tile_path = Path(out_dir, f"{coords.y}.png") + with open(tile_path, "wb") as file: + file.write(rendered_bytes) - try: - subprocess.run(cmd, check=True) - logger.info(f"{prefix} tileset created, elapsed time: {datetime.now() - start}") - except subprocess.CalledProcessError as e: - raise Exception(f"Error during tile generation: {e}") + logger.info(f"{prefix} tileset created, elapsed time: {datetime.now() - start}") if settings.ENABLE_S3_STORAGE: s3 = get_boto3_s3_client() @@ -198,20 +200,7 @@ def generate_xyz_tiles(self, layerset: LayerSet): ## clean up existing tileset if existing_tileset_prefix: - logger.info(f"deleting existing tileset {existing_tileset_prefix}") - if settings.ENABLE_S3_STORAGE: - s3 = get_boto3_s3_client() - response = s3.list_objects_v2( - Bucket=settings.AWS_STORAGE_BUCKET_NAME, Prefix=existing_tileset_prefix - ) - logger.info(f"deleting {len(response['Contents'])} tiles") - for object in response["Contents"]: - s3.delete_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=object["Key"]) - else: - shutil.rmtree( - Path(settings.MEDIA_ROOT, existing_tileset_prefix), - ignore_errors=True, - ) + cleanup_existing_tileset.delay(existing_tileset_prefix) def generate_mosaic_json(self, layerset, trim_all=False): """DEPRECATED: Currently, MosaicJSON is not used anywhere in the app.""" diff --git a/pyproject.toml b/pyproject.toml index f68d42ca..22d92699 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,8 @@ dependencies = [ "lxml", "setuptools", # for celery beat - "pytz" + "pytz", + "rio-tiler>=5.0.3", ] [project.optional-dependencies] From 7a3d7165db356b3b6ff0afcded950efedbd3eeda Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sat, 16 May 2026 23:01:04 -0500 Subject: [PATCH 08/14] save directly to s3 if enabled --- ohmg/georeference/mosaicker.py | 80 +++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/ohmg/georeference/mosaicker.py b/ohmg/georeference/mosaicker.py index d8a739a4..402c4825 100644 --- a/ohmg/georeference/mosaicker.py +++ b/ohmg/georeference/mosaicker.py @@ -1,7 +1,7 @@ +import io import json import logging import os -import shutil from datetime import datetime from glob import glob from pathlib import Path @@ -151,48 +151,56 @@ def generate_xyz_tiles(self, layerset: LayerSet, min_zoom: int = 13, max_zoom: i prefix = f"tiles/{layerset.map.identifier}/{layerset.category.slug}/{random_alnum()}" logger.info(f"creating new tileset {prefix}") + + p = { + 10: False, + 20: False, + 30: False, + 40: False, + 50: False, + 60: False, + 70: False, + 80: False, + 90: False, + } + if settings.ENABLE_S3_STORAGE: - out_path = Path(settings.TEMP_DIR, prefix) - else: - out_path = Path(settings.MEDIA_ROOT, prefix) + s3 = get_boto3_s3_client() with Reader(self.mosaic_vrt.get_path()) as src: - for coords in tms.tiles(*src.geographic_bounds, zooms=range(min_zoom, max_zoom + 1)): + zooms = range(min_zoom, max_zoom + 1) + bounds = src.geographic_bounds + tiles_total_ct = sum(1 for i in tms.tiles(*bounds, zooms=zooms)) + tiles_written_ct = 0 + for coords in tms.tiles(*bounds, zooms=zooms): tile = src.tile(coords.x, coords.y, coords.z) ## only make a tile if there is valid data (skip empty tiles) if tile.data_as_image().any(): rendered_bytes = tile.render() - out_dir = Path(out_path, str(coords.z), str(coords.x)) - out_dir.mkdir(parents=True, exist_ok=True) - tile_path = Path(out_dir, f"{coords.y}.png") - with open(tile_path, "wb") as file: - file.write(rendered_bytes) - - logger.info(f"{prefix} tileset created, elapsed time: {datetime.now() - start}") - - if settings.ENABLE_S3_STORAGE: - s3 = get_boto3_s3_client() - - all_files = [] - for root, dirs, files in os.walk(out_path): - for f in files: - all_files.append( - {"path": Path(root, f), "key": Path(root, f).relative_to(settings.TEMP_DIR)} - ) - - logger.debug( - f"uploading {len(all_files)} tiles to bucket: {settings.AWS_STORAGE_BUCKET_NAME}" - ) - for f in all_files: - s3.upload_file( - f["path"], - settings.AWS_STORAGE_BUCKET_NAME, - str(f["key"]), - ExtraArgs={"ACL": "public-read"}, - ) - - logger.debug("deleting temp local tileset") - shutil.rmtree(out_path) + key = f"{prefix}/{coords.z}/{coords.x}/{coords.y}.png" + if settings.ENABLE_S3_STORAGE: + file_like = io.BytesIO(rendered_bytes) + s3.upload_fileobj( + file_like, + settings.AWS_STORAGE_BUCKET_NAME, + key, + ) + else: + out_root = Path(settings.MEDIA_ROOT, prefix) + out_dir = Path(out_root, str(coords.z), str(coords.x)) + out_dir.mkdir(parents=True, exist_ok=True) + file_path = Path(out_dir, f"{coords.y}.png") + with open(file_path, "wb") as file: + file.write(rendered_bytes) + ## progress logging + tiles_written_ct += 1 + pct = int((tiles_written_ct / tiles_total_ct) * 100) + for k in p.keys(): + if pct > k and not p[k]: + logger.debug(f"{prefix} {k}% written") + p[k] = True + + logger.info(f"{prefix} completed, elapsed time: {datetime.now() - start}") existing_tileset_prefix = layerset.xyz_tiles_prefix layerset.xyz_tiles_prefix = prefix From fa9893d9958e8214dea63c185b748d2fae31ee08 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sat, 16 May 2026 23:01:21 -0500 Subject: [PATCH 09/14] cleanup --- ohmg/conf/settings.py | 1 - ohmg/georeference/mosaicker.py | 10 +++++----- ohmg/georeference/tasks.py | 11 +++++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/ohmg/conf/settings.py b/ohmg/conf/settings.py index 9748d8e9..bfbb262e 100644 --- a/ohmg/conf/settings.py +++ b/ohmg/conf/settings.py @@ -451,7 +451,6 @@ if DEBUG: celery_log_level = "DEBUG" LOGGING["loggers"]["ohmg"]["handlers"].append("console") - LOGGING["loggers"]["ohmg.georeference"]["handlers"].append("console") else: celery_log_level = "INFO" diff --git a/ohmg/georeference/mosaicker.py b/ohmg/georeference/mosaicker.py index 402c4825..58c0aa50 100644 --- a/ohmg/georeference/mosaicker.py +++ b/ohmg/georeference/mosaicker.py @@ -152,7 +152,7 @@ def generate_xyz_tiles(self, layerset: LayerSet, min_zoom: int = 13, max_zoom: i prefix = f"tiles/{layerset.map.identifier}/{layerset.category.slug}/{random_alnum()}" logger.info(f"creating new tileset {prefix}") - p = { + progress_pct = { 10: False, 20: False, 30: False, @@ -177,8 +177,8 @@ def generate_xyz_tiles(self, layerset: LayerSet, min_zoom: int = 13, max_zoom: i ## only make a tile if there is valid data (skip empty tiles) if tile.data_as_image().any(): rendered_bytes = tile.render() - key = f"{prefix}/{coords.z}/{coords.x}/{coords.y}.png" if settings.ENABLE_S3_STORAGE: + key = f"{prefix}/{coords.z}/{coords.x}/{coords.y}.png" file_like = io.BytesIO(rendered_bytes) s3.upload_fileobj( file_like, @@ -195,10 +195,10 @@ def generate_xyz_tiles(self, layerset: LayerSet, min_zoom: int = 13, max_zoom: i ## progress logging tiles_written_ct += 1 pct = int((tiles_written_ct / tiles_total_ct) * 100) - for k in p.keys(): - if pct > k and not p[k]: + for k in progress_pct.keys(): + if pct > k and not progress_pct[k]: logger.debug(f"{prefix} {k}% written") - p[k] = True + progress_pct[k] = True logger.info(f"{prefix} completed, elapsed time: {datetime.now() - start}") diff --git a/ohmg/georeference/tasks.py b/ohmg/georeference/tasks.py index 99155a35..618a5c64 100644 --- a/ohmg/georeference/tasks.py +++ b/ohmg/georeference/tasks.py @@ -87,7 +87,10 @@ def cleanup_existing_tileset(prefix): for object in response["Contents"]: s3.delete_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=object["Key"]) else: - shutil.rmtree( - Path(settings.MEDIA_ROOT, prefix), - ignore_errors=True, - ) + try: + shutil.rmtree( + Path(settings.MEDIA_ROOT, prefix), + ignore_errors=True, + ) + except FileNotFoundError: + pass From 82b89798f580044d67d8b6d79348c0d9b3ff8af0 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sun, 17 May 2026 14:12:19 -0500 Subject: [PATCH 10/14] pull xyz tiles generation to utility function --- ohmg/georeference/mosaicker.py | 59 ++----------------------- ohmg/georeference/utils.py | 78 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 56 deletions(-) create mode 100644 ohmg/georeference/utils.py diff --git a/ohmg/georeference/mosaicker.py b/ohmg/georeference/mosaicker.py index 58c0aa50..3d548044 100644 --- a/ohmg/georeference/mosaicker.py +++ b/ohmg/georeference/mosaicker.py @@ -1,4 +1,3 @@ -import io import json import logging import os @@ -7,20 +6,19 @@ from pathlib import Path from typing import List -import morecantile from django.conf import settings from django.contrib.gis.geos import MultiPolygon, Polygon from django.core.files import File from django.core.files.storage import get_storage_class from osgeo import gdal -from rio_tiler.io import Reader from ohmg.core.models import Layer, LayerSet from ohmg.core.storages import get_file_url -from ohmg.core.utils import get_boto3_s3_client, random_alnum +from ohmg.core.utils import random_alnum from .georeferencer import Georeferencer, VRTHandler from .tasks import cleanup_existing_tileset +from .utils import make_xyz_tiles gdal.SetConfigOption("GDAL_NUM_THREADS", "ALL_CPUS") gdal.SetConfigOption("GDAL_TIFF_INTERNAL_MASK", "YES") @@ -144,63 +142,12 @@ def generate_cog(self, layerset: LayerSet): print(f"completed - elapsed time: {datetime.now() - start}") def generate_xyz_tiles(self, layerset: LayerSet, min_zoom: int = 13, max_zoom: int = 20): - start = datetime.now() - self.generate_mosaic_vrt(layerset) - tms = morecantile.tms.get("WebMercatorQuad") prefix = f"tiles/{layerset.map.identifier}/{layerset.category.slug}/{random_alnum()}" logger.info(f"creating new tileset {prefix}") - progress_pct = { - 10: False, - 20: False, - 30: False, - 40: False, - 50: False, - 60: False, - 70: False, - 80: False, - 90: False, - } - - if settings.ENABLE_S3_STORAGE: - s3 = get_boto3_s3_client() - - with Reader(self.mosaic_vrt.get_path()) as src: - zooms = range(min_zoom, max_zoom + 1) - bounds = src.geographic_bounds - tiles_total_ct = sum(1 for i in tms.tiles(*bounds, zooms=zooms)) - tiles_written_ct = 0 - for coords in tms.tiles(*bounds, zooms=zooms): - tile = src.tile(coords.x, coords.y, coords.z) - ## only make a tile if there is valid data (skip empty tiles) - if tile.data_as_image().any(): - rendered_bytes = tile.render() - if settings.ENABLE_S3_STORAGE: - key = f"{prefix}/{coords.z}/{coords.x}/{coords.y}.png" - file_like = io.BytesIO(rendered_bytes) - s3.upload_fileobj( - file_like, - settings.AWS_STORAGE_BUCKET_NAME, - key, - ) - else: - out_root = Path(settings.MEDIA_ROOT, prefix) - out_dir = Path(out_root, str(coords.z), str(coords.x)) - out_dir.mkdir(parents=True, exist_ok=True) - file_path = Path(out_dir, f"{coords.y}.png") - with open(file_path, "wb") as file: - file.write(rendered_bytes) - ## progress logging - tiles_written_ct += 1 - pct = int((tiles_written_ct / tiles_total_ct) * 100) - for k in progress_pct.keys(): - if pct > k and not progress_pct[k]: - logger.debug(f"{prefix} {k}% written") - progress_pct[k] = True - - logger.info(f"{prefix} completed, elapsed time: {datetime.now() - start}") + make_xyz_tiles(self.mosaic_vrt.get_path(), prefix, min_zoom=min_zoom, max_zoom=max_zoom) existing_tileset_prefix = layerset.xyz_tiles_prefix layerset.xyz_tiles_prefix = prefix diff --git a/ohmg/georeference/utils.py b/ohmg/georeference/utils.py new file mode 100644 index 00000000..3ce4b3a8 --- /dev/null +++ b/ohmg/georeference/utils.py @@ -0,0 +1,78 @@ +import io +import logging +from datetime import datetime +from pathlib import Path +from typing import Union + +import morecantile +from django.conf import settings +from rio_tiler.io import Reader + +from ohmg.core.utils import get_boto3_s3_client + +logger = logging.getLogger(__name__) + + +def make_xyz_tiles( + data_source: Union[str | Path], + prefix: Union[str | Path], + min_zoom: int = 13, + max_zoom: int = 20, +): + start = datetime.now() + + tms = morecantile.tms.get("WebMercatorQuad") + + logger.info(f"creating new tileset {prefix}") + + progress_pct = { + 10: False, + 20: False, + 30: False, + 40: False, + 50: False, + 60: False, + 70: False, + 80: False, + 90: False, + } + + if settings.ENABLE_S3_STORAGE: + s3 = get_boto3_s3_client() + + with Reader(data_source) as src: + zooms = range(min_zoom, max_zoom + 1) + bounds = src.geographic_bounds + tiles_total_ct = sum(1 for i in tms.tiles(*bounds, zooms=zooms)) + tiles_written_ct = 0 + for coords in tms.tiles(*bounds, zooms=zooms): + tile = src.tile(coords.x, coords.y, coords.z) + ## only make a tile if there is valid data (skip empty tiles) + if tile.data_as_image().any(): + rendered_bytes = tile.render() + if settings.ENABLE_S3_STORAGE: + key = f"{prefix}/{coords.z}/{coords.x}/{coords.y}.png" + file_like = io.BytesIO(rendered_bytes) + s3.upload_fileobj( + file_like, + settings.AWS_STORAGE_BUCKET_NAME, + key, + ) + else: + out_root = Path(settings.MEDIA_ROOT, prefix) + out_dir = Path(out_root, str(coords.z), str(coords.x)) + out_dir.mkdir(parents=True, exist_ok=True) + file_path = Path(out_dir, f"{coords.y}.png") + with open(file_path, "wb") as file: + file.write(rendered_bytes) + ## progress logging + tiles_written_ct += 1 + pct = int((tiles_written_ct / tiles_total_ct) * 100) + for k in progress_pct.keys(): + if pct > k and not progress_pct[k]: + logger.debug(f"{prefix} {k}% written") + progress_pct[k] = True + + logger.info(f"{prefix} completed, elapsed time: {datetime.now() - start}") + + return prefix From 9786bddbda86e9860a78a1df80e46a50dc7fdb03 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sun, 17 May 2026 15:17:35 -0500 Subject: [PATCH 11/14] add multiprocessing option to tileset creation --- .../management/commands/mosaic.py | 6 +- ohmg/georeference/mosaicker.py | 24 +++++- ohmg/georeference/utils.py | 76 ++++++++++++++++++- 3 files changed, 97 insertions(+), 9 deletions(-) diff --git a/ohmg/georeference/management/commands/mosaic.py b/ohmg/georeference/management/commands/mosaic.py index 81fd0861..979a86e5 100644 --- a/ohmg/georeference/management/commands/mosaic.py +++ b/ohmg/georeference/management/commands/mosaic.py @@ -38,6 +38,10 @@ def add_arguments(self, parser): "--background", action="store_true", ) + parser.add_argument( + "--multiprocessing", + action="store_true", + ) def handle(self, *args, **options): options = Namespace(**options) @@ -55,7 +59,7 @@ def handle(self, *args, **options): if options.background: create_mosaic_tileset.delay(ls.pk) else: - m.generate_xyz_tiles(ls) + m.generate_xyz_tiles(ls, use_multiprocessing=options.multiprocessing) m.cleanup_files() if options.operation == "generate-cog": diff --git a/ohmg/georeference/mosaicker.py b/ohmg/georeference/mosaicker.py index 3d548044..323ab5eb 100644 --- a/ohmg/georeference/mosaicker.py +++ b/ohmg/georeference/mosaicker.py @@ -18,7 +18,7 @@ from .georeferencer import Georeferencer, VRTHandler from .tasks import cleanup_existing_tileset -from .utils import make_xyz_tiles +from .utils import make_xyz_tiles, make_xyz_tiles_with_multiprocessing gdal.SetConfigOption("GDAL_NUM_THREADS", "ALL_CPUS") gdal.SetConfigOption("GDAL_TIFF_INTERNAL_MASK", "YES") @@ -141,13 +141,29 @@ def generate_cog(self, layerset: LayerSet): print(f"completed - elapsed time: {datetime.now() - start}") - def generate_xyz_tiles(self, layerset: LayerSet, min_zoom: int = 13, max_zoom: int = 20): - self.generate_mosaic_vrt(layerset) + def generate_xyz_tiles( + self, + layerset: LayerSet, + min_zoom: int = 13, + max_zoom: int = 20, + use_multiprocessing: bool = False, + ): + if layerset.mosaic_geotiff: + in_path = f"/vsicurl/{layerset.mosaic_cog_url}" + else: + self.generate_mosaic_vrt(layerset) + in_path = self.mosaic_vrt.get_path() prefix = f"tiles/{layerset.map.identifier}/{layerset.category.slug}/{random_alnum()}" logger.info(f"creating new tileset {prefix}") + logger.info(f"source dataset: {in_path}") - make_xyz_tiles(self.mosaic_vrt.get_path(), prefix, min_zoom=min_zoom, max_zoom=max_zoom) + if use_multiprocessing: + make_xyz_tiles_with_multiprocessing( + in_path, prefix, min_zoom=min_zoom, max_zoom=max_zoom + ) + else: + make_xyz_tiles(in_path, prefix, min_zoom=min_zoom, max_zoom=max_zoom) existing_tileset_prefix = layerset.xyz_tiles_prefix layerset.xyz_tiles_prefix = prefix diff --git a/ohmg/georeference/utils.py b/ohmg/georeference/utils.py index 3ce4b3a8..f0ad6c4d 100644 --- a/ohmg/georeference/utils.py +++ b/ohmg/georeference/utils.py @@ -1,6 +1,8 @@ import io import logging +import os from datetime import datetime +from multiprocessing import Pool from pathlib import Path from typing import Union @@ -12,8 +14,43 @@ logger = logging.getLogger(__name__) +TMS = morecantile.tms.get("WebMercatorQuad") -def make_xyz_tiles( + +def extract_tile_for_multiprocessing(info): + """This is a standalone function to be called within a multiprocessing iteration, + and should only be used in that context. It needs to re-instantiate the Reader + object, and also needs to recreate the s3 client. This is inefficient...""" + + src_url = info.get("src_url") + tile_coords = info.get("tile_coords") + prefix = info.get("prefix") + + with Reader(src_url) as src: + tile = src.tile(tile_coords.x, tile_coords.y, tile_coords.z) + + ## only make a tile if there is valid data (skip empty tiles) + if tile.data_as_image().any(): + rendered_bytes = tile.render() + if settings.ENABLE_S3_STORAGE: + s3 = get_boto3_s3_client() + key = f"{prefix}/{tile_coords.z}/{tile_coords.x}/{tile_coords.y}.png" + file_like = io.BytesIO(rendered_bytes) + s3.upload_fileobj( + file_like, + settings.AWS_STORAGE_BUCKET_NAME, + key, + ) + else: + out_root = Path(settings.MEDIA_ROOT, prefix) + out_dir = Path(out_root, str(tile_coords.z), str(tile_coords.x)) + out_dir.mkdir(parents=True, exist_ok=True) + file_path = Path(out_dir, f"{tile_coords.y}.png") + with open(file_path, "wb") as file: + file.write(rendered_bytes) + + +def make_xyz_tiles_with_multiprocessing( data_source: Union[str | Path], prefix: Union[str | Path], min_zoom: int = 13, @@ -21,7 +58,36 @@ def make_xyz_tiles( ): start = datetime.now() - tms = morecantile.tms.get("WebMercatorQuad") + logger.info(f"creating new tileset with multiprocessing {prefix}") + + with Reader(data_source) as src: + zooms = range(min_zoom, max_zoom + 1) + bounds = src.geographic_bounds + tile_info_list = [ + { + "src_url": data_source, + "tile_coords": i, + "prefix": prefix, + } + for i in TMS.tiles(*bounds, zooms=zooms) + ] + tiles_total_ct = len(tile_info_list) + logger.info(f"{tiles_total_ct} tile coordinate sets") + process_ct = os.cpu_count() + logger.info(f"generating tiles using {process_ct} parallel processes") + with Pool(process_ct) as p: + p.map(extract_tile_for_multiprocessing, tile_info_list) + + logger.info(f"{prefix} completed, elapsed time: {datetime.now() - start}") + + +def make_xyz_tiles( + data_source: Union[str | Path], + prefix: Union[str | Path], + min_zoom: int = 13, + max_zoom: int = 20, +): + start = datetime.now() logger.info(f"creating new tileset {prefix}") @@ -43,9 +109,11 @@ def make_xyz_tiles( with Reader(data_source) as src: zooms = range(min_zoom, max_zoom + 1) bounds = src.geographic_bounds - tiles_total_ct = sum(1 for i in tms.tiles(*bounds, zooms=zooms)) + tile_coords = list(TMS.tiles(*bounds, zooms=zooms)) + tiles_total_ct = len(tile_coords) + logger.info(f"{tiles_total_ct} tile coordinate sets") tiles_written_ct = 0 - for coords in tms.tiles(*bounds, zooms=zooms): + for coords in TMS.tiles(*bounds, zooms=zooms): tile = src.tile(coords.x, coords.y, coords.z) ## only make a tile if there is valid data (skip empty tiles) if tile.data_as_image().any(): From d6b649f66ae48c966ac66671d82f55485ac48232 Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sun, 17 May 2026 21:57:28 -0500 Subject: [PATCH 12/14] update uv.lock --- uv.lock | 227 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) diff --git a/uv.lock b/uv.lock index 146b07a8..4bcf06a7 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,15 @@ version = 1 revision = 3 requires-python = "==3.10.*" +[[package]] +name = "affine" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/98/d2f0bb06385069e799fc7d2870d9e078cfa0fa396dc8a2b81227d0da08b9/affine-2.4.0.tar.gz", hash = "sha256:a24d818d6a836c131976d22f8c27b8d3ca32d0af64c1d8d29deb7bafa4da1eea", size = 17132, upload-time = "2023-01-19T23:44:30.696Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/f7/85273299ab57117850cc0a936c64151171fac4da49bc6fba0dad984a7c5f/affine-2.4.0-py3-none-any.whl", hash = "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92", size = 15662, upload-time = "2023-01-19T23:44:28.833Z" }, +] + [[package]] name = "amqp" version = "5.3.1" @@ -14,6 +23,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, ] +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup" }, + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + [[package]] name = "asgiref" version = "3.8.1" @@ -26,6 +49,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/e3/893e8757be2612e6c266d9bb58ad2e3651524b5b40cf56761e985a28b13e/asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47", size = 23828, upload-time = "2024-03-22T14:39:34.521Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "beautifulsoup4" version = "4.14.3" @@ -76,6 +108,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/32/8a4a0447432425cd2f772c757d988742685f46796cf5d68aeaf6bcb6bc37/botocore-1.42.27-py3-none-any.whl", hash = "sha256:d51fb3b8dd1a944c8d238d2827a0dd6e5528d6da49a3bd9eccad019c533e4c9c", size = 14555236, upload-time = "2026-01-13T20:34:55.918Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/e2/85f227594656000ff4d8adadae91a21f536d4a84c6c716a86bd6685874be/cachetools-7.1.1.tar.gz", hash = "sha256:27bdf856d68fd3c71c26c01b5edc312124ed427524d1ddb31aa2b7746fe20d4b", size = 40202, upload-time = "2026-05-03T20:00:29.391Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/0f/f897abe4ea0a8c408ae65c8c83bffab4936ad65d6032d4fb4cd35bbdc3ee/cachetools-7.1.1-py3-none-any.whl", hash = "sha256:0335cd7a0952d2b22327441fb0628139e234c565559eeb91a8a4ac7551c5353d", size = 16775, upload-time = "2026-05-03T20:00:27.857Z" }, +] + [[package]] name = "celery" version = "5.6.0" @@ -212,6 +253,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, ] +[[package]] +name = "cligj" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ea/0d/837dbd5d8430fd0f01ed72c4cfb2f548180f4c68c635df84ce87956cff32/cligj-0.7.2.tar.gz", hash = "sha256:a4bc13d623356b373c2c27c53dbd9c68cae5d526270bfa71f6c6fa69669c6b27", size = 9803, upload-time = "2021-05-28T21:23:27.935Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/86/43fa9f15c5b9fb6e82620428827cd3c284aa933431405d1bcf5231ae3d3e/cligj-0.7.2-py3-none-any.whl", hash = "sha256:c1ca117dbce1fe20a5809dc96f01e1c2840f6dcc939b3ddbb1111bf330ba82df", size = 7069, upload-time = "2021-05-28T21:23:26.877Z" }, +] + +[[package]] +name = "color-operations" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/d5/8daa1179809f0d8eab39bd83ce8131e84691eb6ba55f19b7b365a822fea3/color_operations-0.2.0.tar.gz", hash = "sha256:f1bff5cff5992ec7d240f1979320a981f2e9f77d983e9298291e02f3ffaac9bf", size = 18042, upload-time = "2025-03-27T08:42:14.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/28/d2d3c8399f10dbd876392e7b513efb84462a700b792c2a10c0bd5491744d/color_operations-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b5119c3136f0f18e470ac7ff95b0a92899b450d63a6bbe518f1b0ca6e2c88685", size = 86568, upload-time = "2025-03-27T09:18:09.431Z" }, + { url = "https://files.pythonhosted.org/packages/b6/24/7ef96b436f6c12ba4dc6c7879b9022b70cb3b7ecd687325064d913fa49ae/color_operations-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f92bdf4341c0516e1779ac3e3db83b871af2d989b210b6bd713ef46ead5705dd", size = 50995, upload-time = "2025-03-27T09:18:10.908Z" }, + { url = "https://files.pythonhosted.org/packages/03/97/1aa16eac64c8e08cd21c769c79574e4049769dee2f126bd3f75c92ac9c32/color_operations-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e0e1da1f56cb9efba786fa649fb8e02524269e1995cdb7b916674a02b6d3e66c", size = 49301, upload-time = "2025-03-27T09:18:12.235Z" }, + { url = "https://files.pythonhosted.org/packages/7e/f8/289d0b28404785d3e11005359717f11e1fb050ad78c9cc563dc18da3a7b7/color_operations-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3bfe5ec92964140eaf37969d22f6b1211bbe86fee006c962d178019f0c80d504", size = 180027, upload-time = "2025-03-27T09:18:13.269Z" }, + { url = "https://files.pythonhosted.org/packages/c1/c2/8ce0fbbb3aba230ed63763118fdf4d0fd9152f8a2716b0428dfc44c2930c/color_operations-0.2.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8890741acc4fd31a4f749f94c8ec85b2e10e1a3369f05d1ae1e92ebfe7c638ba", size = 186503, upload-time = "2025-03-27T09:18:14.621Z" }, + { url = "https://files.pythonhosted.org/packages/4d/39/c42d1488c200d1c5814deea01aa1b6ea94dec53e546a94abc58c21a34a3e/color_operations-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:f31b8351772b32215e67d7dda8ceafe26e2c80412731c23b4baa2962af37c960", size = 133485, upload-time = "2025-03-27T09:18:15.729Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -530,6 +600,43 @@ version = "3.5.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/8b/9f/c9fe38625e8d9ea814d4c3935773341cffb226b12ce2356bad80a9c0e4e8/GDAL-3.5.3.tar.gz", hash = "sha256:ff11ff32b400086930e0e0d15e35da598dfae548423512759e111ea9ad65b19b", size = 756836, upload-time = "2022-10-31T16:40:05.87Z" } +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + [[package]] name = "humanize" version = "4.10.0" @@ -620,6 +727,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/08/83871f3c50fc983b88547c196d11cf8c3340e37c32d2e9d6152abe2c61f7/Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803", size = 106349, upload-time = "2024-08-16T15:55:16.176Z" }, ] +[[package]] +name = "morecantile" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cachetools" }, + { name = "pydantic" }, + { name = "pyproj" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/fc/152907ef045cb179ad2246aa892c0aefb6e0b5251930a6e408791f68190a/morecantile-4.3.0.tar.gz", hash = "sha256:aa683415f0e6d07f804912ddf15f5a4a96e23281205776e12be5b323b75ca447", size = 31818, upload-time = "2023-07-11T20:41:52.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/ca/40f682bb6b910d8d5eed990c9f00a7b9059172cc2971706a8022246e060f/morecantile-4.3.0-py3-none-any.whl", hash = "sha256:610f1dcc3ae0a99f0a0e6c05e18508d0e9e26b53279e06a65150939e2f54963b", size = 36643, upload-time = "2023-07-11T20:41:51.178Z" }, +] + [[package]] name = "natsort" version = "8.4.0" @@ -638,6 +760,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "numexpr" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/2f/fdba158c9dbe5caca9c3eca3eaffffb251f2fb8674bf8e2d0aed5f38d319/numexpr-2.14.1.tar.gz", hash = "sha256:4be00b1086c7b7a5c32e31558122b7b80243fe098579b170967da83f3152b48b", size = 119400, upload-time = "2025-10-13T16:17:27.351Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/91/ccd504cbe5b88d06987c77f42ba37a13ef05065fdab4afe6dcfeb2961faf/numexpr-2.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d0fab3fd06a04f6b86102552b26aa5d85e20ac7d8296c15764c726eeabae6cc8", size = 163200, upload-time = "2025-10-13T16:16:25.47Z" }, + { url = "https://files.pythonhosted.org/packages/f3/89/6b07977baf2af75fb6692f9e7a1fb612a15f600fc921f3f565366de01f4a/numexpr-2.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:64ae5dfd62d74a3ef82fe0b37f80527247f3626171ad82025900f46ffca4b39a", size = 152085, upload-time = "2025-10-13T16:16:29.508Z" }, + { url = "https://files.pythonhosted.org/packages/28/c2/c5775541256c4bf16b4d88fa1cffa74a0126703e513093c8774d911b0bb7/numexpr-2.14.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:955c92b064f9074d2970cf3138f5e3b965be673b82024962ed526f39bc25a920", size = 449435, upload-time = "2025-10-13T16:13:16.257Z" }, + { url = "https://files.pythonhosted.org/packages/34/d4/d1a410901c620f7a6a3c5c2b1fc9dab22170be05a89d2c02ae699e27bd3f/numexpr-2.14.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75440c54fc01e130396650fdf307aa9d41a67dc06ddbfb288971b591c13a395b", size = 440197, upload-time = "2025-10-13T16:14:44.109Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c8/fa85f0cc5c39db587ba4927b862a92477c017ee8476e415e8120a100457b/numexpr-2.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dde9fa47ed319e1e1728940a539df3cb78326b7754bc7c6ab3152afc91808f9b", size = 1414125, upload-time = "2025-10-13T16:13:19.882Z" }, + { url = "https://files.pythonhosted.org/packages/08/72/a58ddc05e0eabb3fa8d3fcd319f3d97870e6b41520832acfd04a6734c2c0/numexpr-2.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76db0bc6267e591ab9c4df405ffb533598e4c88239db7338d11ae9e4b368a85a", size = 1463041, upload-time = "2025-10-13T16:14:47.502Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c5/bdd1862302bb71a78dba941eaf7060e1274f1cf6af2d1b0f1880bfcb289b/numexpr-2.14.1-cp310-cp310-win32.whl", hash = "sha256:0d1dcbdc4d0374c0d523cee2f94f06b001623cbc1fd163612841017a3495427c", size = 166833, upload-time = "2025-10-13T16:17:03.543Z" }, + { url = "https://files.pythonhosted.org/packages/18/af/26773a246716922794388786529e5640676399efabb0ee217ce034df9d27/numexpr-2.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:823cd82c8e7937981339f634e7a9c6a92cb2d0b9d0a5cf627a5e394fffc05377", size = 160068, upload-time = "2025-10-13T16:17:05.191Z" }, +] + [[package]] name = "numpy" version = "1.24.2" @@ -695,6 +836,7 @@ dependencies = [ { name = "python-slugify" }, { name = "pytz" }, { name = "requests" }, + { name = "rio-tiler" }, { name = "setuptools" }, { name = "sorl-thumbnail" }, ] @@ -754,6 +896,7 @@ requires-dist = [ { name = "python-slugify" }, { name = "pytz" }, { name = "requests", specifier = "==2.28.2" }, + { name = "rio-tiler", specifier = ">=5.0.3" }, { name = "ruff", marker = "extra == 'dev'" }, { name = "setuptools" }, { name = "sorl-thumbnail", specifier = "==12.8.0" }, @@ -920,6 +1063,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/2c/5b079febdc65e1c3fb2729bf958d18b45be7113828528e8a0b5850dd819a/pymdown_extensions-10.21-py3-none-any.whl", hash = "sha256:91b879f9f864d49794c2d9534372b10150e6141096c3908a455e45ca72ad9d3f", size = 268877, upload-time = "2026-02-15T20:44:05.464Z" }, ] +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "pyproj" +version = "3.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/10/a8480ea27ea4bbe896c168808854d00f2a9b49f95c0319ddcbba693c8a90/pyproj-3.7.1.tar.gz", hash = "sha256:60d72facd7b6b79853f19744779abcd3f804c4e0d4fa8815469db20c9f640a47", size = 226339, upload-time = "2025-02-16T04:28:46.621Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/a3/c4cd4bba5b336075f145fe784fcaf4ef56ffbc979833303303e7a659dda2/pyproj-3.7.1-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:bf09dbeb333c34e9c546364e7df1ff40474f9fddf9e70657ecb0e4f670ff0b0e", size = 6262524, upload-time = "2025-02-16T04:27:19.725Z" }, + { url = "https://files.pythonhosted.org/packages/40/45/4fdf18f4cc1995f1992771d2a51cf186a9d7a8ec973c9693f8453850c707/pyproj-3.7.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6575b2e53cc9e3e461ad6f0692a5564b96e7782c28631c7771c668770915e169", size = 4665102, upload-time = "2025-02-16T04:27:24.428Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d2/360eb127380106cee83569954ae696b88a891c804d7a93abe3fbc15f5976/pyproj-3.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cb516ee35ed57789b46b96080edf4e503fdb62dbb2e3c6581e0d6c83fca014b", size = 9432667, upload-time = "2025-02-16T04:27:27.04Z" }, + { url = "https://files.pythonhosted.org/packages/76/a5/c6e11b9a99ce146741fb4d184d5c468446c6d6015b183cae82ac822a6cfa/pyproj-3.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e47c4e93b88d99dd118875ee3ca0171932444cdc0b52d493371b5d98d0f30ee", size = 9259185, upload-time = "2025-02-16T04:27:30.35Z" }, + { url = "https://files.pythonhosted.org/packages/41/56/a3c15c42145797a99363fa0fdb4e9805dccb8b4a76a6d7b2cdf36ebcc2a1/pyproj-3.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3e8d276caeae34fcbe4813855d0d97b9b825bab8d7a8b86d859c24a6213a5a0d", size = 10469103, upload-time = "2025-02-16T04:27:33.542Z" }, + { url = "https://files.pythonhosted.org/packages/ef/73/c9194c2802fefe2a4fd4230bdd5ab083e7604e93c64d0356fa49c363bad6/pyproj-3.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f173f851ee75e54acdaa053382b6825b400cb2085663a9bb073728a59c60aebb", size = 10401391, upload-time = "2025-02-16T04:27:36.051Z" }, + { url = "https://files.pythonhosted.org/packages/c5/1d/ce8bb5b9251b04d7c22d63619bb3db3d2397f79000a9ae05b3fd86a5837e/pyproj-3.7.1-cp310-cp310-win32.whl", hash = "sha256:f550281ed6e5ea88fcf04a7c6154e246d5714be495c50c9e8e6b12d3fb63e158", size = 5869997, upload-time = "2025-02-16T04:27:38.302Z" }, + { url = "https://files.pythonhosted.org/packages/09/6a/ca145467fd2e5b21e3d5b8c2b9645dcfb3b68f08b62417699a1f5689008e/pyproj-3.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:3537668992a709a2e7f068069192138618c00d0ba113572fdd5ee5ffde8222f3", size = 6278581, upload-time = "2025-02-16T04:27:41.051Z" }, +] + +[[package]] +name = "pystac" +version = "1.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/e6/efbc20dbc94ad7ed18fe11a4208103a509384ffcccd9bdc27953b725e686/pystac-1.14.3.tar.gz", hash = "sha256:24f92d6f301371859aa0abc1bbe7b1523a603e1184a6d139ecb323967c2c9bb3", size = 164205, upload-time = "2026-01-09T12:38:42.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/b4/a9430e72bfc3c458e1fcf8363890994e483052ab052ed93912be4e5b32c8/pystac-1.14.3-py3-none-any.whl", hash = "sha256:2f60005f521d541fb801428307098f223c14697b3faf4d2f0209afb6a43f39e5", size = 208506, upload-time = "2026-01-09T12:38:40.721Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -991,6 +1174,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, ] +[[package]] +name = "rasterio" +version = "1.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "affine" }, + { name = "attrs" }, + { name = "certifi" }, + { name = "click" }, + { name = "click-plugins" }, + { name = "cligj" }, + { name = "numpy" }, + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ec/fa/fce8dc9f09e5bc6520b6fc1b4ecfa510af9ca06eb42ad7bdff9c9b8989d0/rasterio-1.4.4.tar.gz", hash = "sha256:c95424e2c7f009b8f7df1095d645c52895cd332c0c2e1b4c2e073ea28b930320", size = 445004, upload-time = "2025-12-12T18:01:08.971Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/24/eedb9dfed1706c696b4f43ba9b85e830ce332f4f57ffcb7b6a4c4e66ade9/rasterio-1.4.4-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:35401e84d4d0b239bd62b33d4ee68d7bb13b47c3b41078f4aad7ad7964e61c73", size = 21127567, upload-time = "2025-12-12T17:58:42.664Z" }, + { url = "https://files.pythonhosted.org/packages/67/5f/482a24bf75bcd48236cd223d037f22abc6c08da6961e390e6a35249a9f58/rasterio-1.4.4-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:1f17fc9608b6b6666894a04e0118d3329e831a6347bc3650584d247a9d476fdd", size = 25735929, upload-time = "2025-12-12T17:58:46.503Z" }, + { url = "https://files.pythonhosted.org/packages/b0/da/b988ffb1bb37cc4cb8a028447ab654a16dbac0b339d977fb9c8adc5bd995/rasterio-1.4.4-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:1f0edb8cb30ff8f5be341583f69c115b7c36ad52bbbe7582345d32af115bc6b3", size = 34040467, upload-time = "2025-12-12T17:58:50.109Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0a/2eace22e990203d47a8fed4b174b87be50281bf3f5b2509cf3700036cbcc/rasterio-1.4.4-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:5197da0e3dd09907bdb343717a49e8fb5229ffdbff0e583b874959ec41fa9558", size = 35339947, upload-time = "2025-12-12T17:58:53.269Z" }, + { url = "https://files.pythonhosted.org/packages/40/e5/16acecbbaedd820c5d71f99f3bab73c00455078a414b511f6854364d3e1e/rasterio-1.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:15109134c7b4770e6aeb8d45dc52c2603824805ba734323268a44f5a81756a7a", size = 25708679, upload-time = "2025-12-12T17:58:56.661Z" }, +] + [[package]] name = "rcssmin" version = "1.1.1" @@ -1030,6 +1236,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, ] +[[package]] +name = "rio-tiler" +version = "5.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cachetools" }, + { name = "color-operations" }, + { name = "httpx" }, + { name = "morecantile" }, + { name = "numexpr" }, + { name = "numpy" }, + { name = "pydantic" }, + { name = "pystac" }, + { name = "rasterio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/63/61b3579543b6ee1a6eaf38a7d4375f5967d8810bcbfecd547a04bfdb0b19/rio_tiler-5.0.3.tar.gz", hash = "sha256:d66ec43dbb7153344b3c30baf882567bdaf249215a98caafc53d0158f7aff4a7", size = 137019, upload-time = "2023-07-18T20:53:54.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/9e/01c3d3d5d348db7f8c70816789d951dbf65ebaf5a89da08f0ef24f56c6bd/rio_tiler-5.0.3-py3-none-any.whl", hash = "sha256:5b19be51fd075ce1e3f24a6ff922ac29f80af469c58eeb529ece64eff094591e", size = 210660, upload-time = "2023-07-18T20:53:56.852Z" }, +] + [[package]] name = "rjsmin" version = "1.2.1" From 5f0d69910e6e753d216944058cebf9ef4fa9e57f Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sun, 17 May 2026 23:07:29 -0500 Subject: [PATCH 13/14] update version to 0.2.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 22d92699..87c61dc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "ohmg" description = "A web georeferencing application for historical maps." -version = "0.1.0-beta" +version = "0.2.0" readme = "README.md" requires-python = ">=3.10,<3.11" license = {file = "LICENSE"} From 8058c2cd7bbb74de8e855e31ac023ee662149a4c Mon Sep 17 00:00:00 2001 From: Adam Cox Date: Sun, 17 May 2026 23:10:39 -0500 Subject: [PATCH 14/14] update uv.lock --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 4bcf06a7..df4ef8fa 100644 --- a/uv.lock +++ b/uv.lock @@ -804,7 +804,7 @@ wheels = [ [[package]] name = "ohmg" -version = "0.1.0b0" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },