Importers

ohmg/accounts/migrations/0003_alter_apikey_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.18 on 2024-04-25 15:52
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0002_apikey'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='apikey',
+            options={'verbose_name': 'API Key'},
+        ),
+    ]

ohmg/accounts/models.py

@@ -51,6 +51,9 @@ def generate_key():
 
 class APIKey(models.Model):
 
+    class Meta:
+        verbose_name = "API Key"
+
     value = models.CharField(
         primary_key=True,
         default=generate_key,

ohmg/content/models.py

@@ -1,45 +1,8 @@
-'''
-This is a step toward moving database models from the loc_insurancemaps app into
-this content app. At present, these are not Django models or even Django proxy models,
-just light-weight objects that are instantiated through the Volume and related
-models. This will allow the codebase to slowly evolve before actually changing any
-database content and running migrations.
-
-The eventual migration plan is this:
-
-ohmg.loc_insurancemaps.models.Volume        --> content.models.Map
-ohmg.loc_insurancemaps.models.Sheet         --> content.models.Resource
-
-new model (idea)                            --> content.models.ItemConfigPreset
-    This would allow an extraction of Sanborn-specific properties vs. generic item
-    uploads. Still unclear exactly what to call this, or everything that it would have.
-    Think about this more when the Map model is created, and a hard-look is made at its
-    attributes.
-'''
-
-import os
-import logging
-from datetime import datetime
-
 from django.contrib.gis.db import models
-from django.core.files import File
-from django.core.files.base import ContentFile
 
 from markdownx.models import MarkdownxField
 
 from ohmg.core.utils import slugify
-from ohmg.loc_insurancemaps.utils import (
-    get_jpg_from_jp2_url,
-)
-from ohmg.core.storages import OverwriteStorage
-from ohmg.core.renderers import generate_document_thumbnail_content
-from ohmg.loc_insurancemaps.models import Volume
-
-logger = logging.getLogger(__name__)
-
-
-class Map(object):
-    pass
 
 
 class Page(models.Model):
@@ -67,67 +30,3 @@ def __str__(self):
     def save(self, *args, **kwargs):
         self.slug = slugify(self.title)
         return super(Page, self).save(*args, **kwargs)
-
-
-class Resource(object):
-    """Resources represent the individual source files that are directly attached to Maps.
-    They represent pages in an atlas or even just a single scan of a map."""
-    volume = models.ForeignKey(Volume, on_delete=models.CASCADE)
-    page_number = models.CharField(max_length=10, null=True, blank=True)
-    file = models.FileField(
-        upload_to='resources',
-        null=True,
-        blank=True,
-        max_length=255,
-        storage=OverwriteStorage(),
-    )
-    thumbnail = models.FileField(
-        upload_to='thumbnails',
-        null=True,
-        blank=True,
-        max_length=255,
-        storage=OverwriteStorage(),
-    )
-    source_url = models.CharField(
-        max_length=255,
-        null=True,
-        blank=True,
-        help_text="Storing a source_url allows this resource to be downloaded at any point after "\
-            "the instance has been created."
-    )
-    load_date = models.DateTimeField(null=True, blank=True)
-
-    @property
-    def name(self):
-        return f"{self.volume.__str__()} p{self.page_number}"
-
-    def __str__(self):
-        return self.name
-
-    def download_file(self):
-
-        log_prefix = f"{self.__str__()} |"
-        logger.info(f"{log_prefix} start load")
-
-        if not self.source_url:
-            logger.warn(f"{log_prefix} no source_url - cancelling download")
-            return
-
-        if not self.file:
-            jpg_path = get_jpg_from_jp2_url(self.source_url)
-            with open(jpg_path, "rb") as new_file:
-                self.file.save(f"{slugify(self.name)}.jpg", File(new_file))
-            os.remove(jpg_path)
-
-        self.load_date = datetime.now()
-        self.save()
-
-    def set_thumbnail(self):
-        if self.file is not None:
-            if self.thumbnail:
-                self.thumbnail.delete()
-            path = self.file.path
-            name = os.path.splitext(os.path.basename(path))[0]
-            content = generate_document_thumbnail_content(path)
-            tname = f"{name}-res-thumb.jpg"
-            self.thumbnail.save(tname, ContentFile(content))

ohmg/core/context_processors.py

@@ -54,11 +54,18 @@ def generate_ohmg_context(request):
         "change_avatar": reverse('avatar_change'),
     }
 
+    csrf_token = csrf.get_token(request)
     return {
         "titiler_host": settings.TITILER_HOST,
         "mapbox_api_token": settings.MAPBOX_API_TOKEN,
-        "ohmg_api_headers": {'X-API-Key': settings.OHMG_API_KEY},
-        "csrf_token": csrf.get_token(request),
+        "ohmg_api_headers": {
+            'X-API-Key': settings.OHMG_API_KEY,
+        },
+        "ohmg_post_headers": {
+            'Content-Type': 'application/json;charset=utf-8',
+            'X-CSRFToken': csrf_token,
+        },
+        "csrf_token": csrf_token,
         "session_length": settings.GEOREFERENCE_SESSION_LENGTH,
         "on_mobile": on_mobile(request)['on_mobile'],
         "user": user_info_from_request(request),

ohmg/loc_insurancemaps/utils.py → ohmg/core/importers/loc_sanborn.py

@@ -5,53 +5,22 @@
 import logging
 import requests
 from datetime import datetime
-from PIL import Image
 
 from django.conf import settings
 from django.urls import reverse
 
 from ohmg.core.utils import (
-    download_image,
     full_capitalize,
 )
+from ohmg.loc_insurancemaps.models import Volume
+from ohmg.georeference.models import AnnotationSet, SetCategory
 
 from ohmg.core.utils import (
     STATE_CHOICES,
 )
 
 logger = logging.getLogger(__name__)
 
-def convert_img_format(input_img, format="JPEG"):
-
-    ext_map = {"PNG":".png", "JPEG":".jpg", "TIFF": ".tif"}
-    ext = os.path.splitext(input_img)[1]
-
-    outpath = input_img.replace(ext, ext_map[format])
-
-    img = Image.open(input_img)
-    img.save(outpath, format=format)
-
-    return outpath
-
-def get_jpg_from_jp2_url(jp2_url):
-
-    temp_img_dir = os.path.join(settings.CACHE_DIR, "img")
-    if not os.path.isdir(temp_img_dir):
-        os.mkdir(temp_img_dir)
-
-    tmp_path = os.path.join(temp_img_dir, jp2_url.split("/")[-1])
-
-    tmp_path = download_image(jp2_url, tmp_path)
-    print(tmp_path)
-    if tmp_path is None:
-        return
-
-    # convert the downloaded jp2 to jpeg (needed for OpenLayers static image)
-    jpg_path = convert_img_format(tmp_path, format="JPEG")
-    os.remove(tmp_path)
-
-    return jpg_path
-
 def filter_volumes_for_use(volumes):
     """
     This is the primary filter function that is applied to a set of volumes
@@ -80,15 +49,31 @@ def filter_volumes_for_use(volumes):
     return volumes
 
 def load_city_name_misspellings(state_name):
+    """ Currently unused as this is a small lookup and can live in this
+    file for now. May be good to move it back out to a separate JSON file eventually. """
+
+    # lookup = {}
+    # file_path = os.path.join(settings.BASE_DIR, "loc_insurancemaps", "reference_data", "city-name-misspellings.json")
+    # if not os.path.isfile(file_path):
+    #     return lookup
+    # with open(file_path, "r") as o:
+    #     data = json.load(o)
+    # lookup = data.get(state_name.lower(), {})
+
+    lookup = {
+        "louisiana": {
+            "Jeannerette": "Jeanerette",
+            "De Quincy": "DeQuincy",
+            "De Ridder": "DeRidder",
+            "Keatchie": "Keachi",
+            "Saint Rose": "St. Rose",
+            "Saint Martinville": "St. Martinville",
+            "Saint Francisville": "St. Francisville"
+        }
+    }
 
-    lookup = {}
-    file_path = os.path.join(settings.BASE_DIR, "loc_insurancemaps", "reference_data", "city-name-misspellings.json")
-    if not os.path.isfile(file_path):
-        return lookup
-    with open(file_path, "r") as o:
-        data = json.load(o)
-    lookup = data.get(state_name.lower(), {})
-    return lookup
+    l = lookup.get(state_name, {})
+    return l
 
 def unsanitize_name(state, name):
     """must 'uncorrect' names from the interface which need to be passed to 
@@ -102,6 +87,71 @@ def unsanitize_name(state, name):
 
     return rev.get(name, name)
 
+def import_all_available_volumes(state, apply_filter=True, verbose=False):
+    """Preparatory step that runs through all cities in the provided
+    state, filters the available volumes for those cities, and then
+    imports each one to create a new Volume object."""
+
+    lc = LOCConnection(delay=0, verbose=verbose)
+    cities = lc.get_city_list_by_state(state)
+
+    volumes = []
+    for city in cities:
+        lc.reset()
+        city = unsanitize_name(state, city[0])
+        vols = lc.get_volume_list_by_city(city, state)
+        if apply_filter is True:
+            vols = filter_volumes_for_use(vols)
+            volumes += [i for i in vols if i['include'] is True]
+        else:
+            volumes += vols
+
+    for volume in volumes:
+        try:
+            Volume.objects.get(pk=volume['identifier'])
+        except Volume.DoesNotExist:
+            import_volume(volume['identifier'])
+
+def import_volume(identifier, locale=None, dry_run=False):
+
+    try:
+        volume = Volume.objects.get(pk=identifier)
+        volume.locales.set([locale])
+        volume.update_place_counts()
+
+        # make sure a main-content layerset exists for this volume
+        main_ls, _ = AnnotationSet.objects.get_or_create(
+            category=SetCategory.objects.get(slug="main-content"),
+            volume=volume,
+        )
+        return volume
+    except Volume.DoesNotExist:
+        pass
+
+    lc = LOCConnection(delay=0, verbose=True)
+    response = lc.get_item(identifier)
+    if response.get("status") == 404:
+        return None
+
+    parsed = LOCParser(item=response['item'])
+    volume_kwargs = parsed.volume_kwargs()
+
+    # add resources to args, not in item (they exist adjacent)
+    volume_kwargs["lc_resources"] = response['resources']
+
+    if dry_run:
+        return volume_kwargs
+    else:
+        volume = Volume.objects.create(**volume_kwargs)
+        volume.locales.add(locale)
+        volume.update_place_counts()
+        # make sure a main-content layerset exists for this volume
+        main_ls, _ = AnnotationSet.objects.get_or_create(
+            category=SetCategory.objects.get(slug="main-content"),
+            volume=volume,
+        )
+        return volume
+
 
 class LOCParser(object):
 
@@ -274,8 +324,6 @@ def create_item_title(self):
 
     def serialize_to_volume(self):
 
-        from .models import Volume
-
         try:
             v = Volume.objects.get(identifier=self.identifier)
             status = v.status

ohmg/loc_insurancemaps/management/commands/volume.py → ohmg/core/management/commands/volume.py

@@ -4,6 +4,7 @@
 from django.core.management.base import BaseCommand
 from django.contrib.auth import get_user_model
 
+from ohmg.core.importers.loc_sanborn import import_volume
 from ohmg.loc_insurancemaps.models import Volume
 from ohmg.places.models import Place
 from ohmg.places.management.utils import reset_volume_counts
@@ -146,7 +147,7 @@ def get_locale(locale_slug):
                         to_load.append((row["identifier"], locale))
 
             for identifier, locale in to_load:
-                vol = Volume().import_volume(
+                vol = import_volume(
                     identifier,
                     locale=locale,
                     dry_run=options['dry_run']