NO TICKET: Transfer

.github/workflows/format_code.yml

@@ -1,6 +1,8 @@
 name: Code Format
 on:
   push:
+    branches-ignore:
+      - transfer
 permissions:
   contents: read
 jobs:

Procfile

@@ -1 +1 @@
-web: gunicorn -b :8080 transfers.entrypoint:app -k uvicorn.workers.UvicornWorker
+web: python3 -m transfers.transfer

core/initializers.py

@@ -50,7 +50,7 @@ def init_hypertables():
     # session.close()
 
 
-def init_lexicon(path=None):
+def init_lexicon(path: str = None) -> None:
     if path is None:
         path = Path(__file__).parent / "lexicon.json"
 

db/location.py

@@ -38,9 +38,7 @@
 class Location(Base, AutoBaseMixin, ReleaseMixin):
     __versioned__ = {}
 
-    nma_pk_location: Mapped[UUID] = mapped_column(
-        String(36), nullable=True, unique=True
-    )
+    nma_pk_location: Mapped[UUID] = mapped_column(String(36), nullable=True)
     description: Mapped[str] = mapped_column
     # name: Mapped[str] = mapped_column(String(255), nullable=True)
     point: Mapped[WKBElement] = mapped_column(

pyproject.toml

@@ -54,7 +54,6 @@ dependencies = [
     "multidict==6.6.3",
     "numpy==2.3.2",
     "packaging==25.0",
-    "pact-python>=2.3.3",
     "pandas==2.3.2",
     "pandas-stubs==2.3.0.250703",
     "pg8000==1.31.4",

services/lexicon_helper.py

@@ -58,8 +58,8 @@ def add_lexicon_term(
             )
             audit_add(user, category)
             session.add(category)
-            session.commit()
-            session.flush()
+            # session.commit()
+            # session.flush()
 
             db_categories.append(category)
 

transfers/asset_transfer.py

@@ -32,7 +32,8 @@
     get_storage_bucket,
     get_storage_client,
 )
-from transfers.util import get_valid_things, logger
+from transfers.util import get_valid_things, read_csv
+from transfers.logger import logger
 
 
 def transfer_assets(session: Session) -> None:
@@ -41,21 +42,34 @@ def transfer_assets(session: Session) -> None:
     bucket = get_storage_bucket(client)
     logger.info(f"Using bucket {bucket.name}")
 
+    well_photos = read_csv("WellPhotos")
     # for name in ['AR0001']: # for testing
     for thing in get_valid_things(session):
-        name = thing.name
-        # find images in temp bucket
-        logger.info(f"Processing PointID: {thing.name}")
-        blobs = bucket.list_blobs(prefix=f"nma-photos/{thing.name}")
-        # move blobs from temp to assets bucket
-        for srcblob in blobs:
-            f = srcblob.download_as_bytes()
-            head, filename = srcblob.name.split("/")
+        photos = well_photos[well_photos["PointID"] == thing.name]
+        if photos.empty:
+            photos = well_photos[well_photos["PointID"] == thing.name.replace("-", "")]
+            if photos.empty:
+                logger.info(f"No photos found for PointID: {thing.name}")
+                continue
 
+        for i, row in enumerate(photos.itertuples()):
+            photo_path = row.OLEPath
+            srcblob = bucket.get_blob(f"nma-photos/{photo_path}")
+            if not srcblob:
+                logger.critical(
+                    f"No photo found for PointID: {thing.name}, {photo_path}"
+                )
+                continue
+
+            head, filename = srcblob.name.split("/")
+            f = srcblob.download_as_bytes()
             ff = UploadFile(file=io.BytesIO(f), filename=filename, size=len(f))
 
             uri, blob_name = gcs_upload(ff, bucket)
             add_asset(session, ff, filename, thing.id, uri, blob_name)
+            logger.info(
+                f"Added asset thing.id={thing.id} thing={thing.name} uri: {uri}"
+            )
 
 
 def transfer_assets_testing(session: Session) -> None:

transfers/contact_transfer.py

@@ -15,9 +15,9 @@
 # ===============================================================================
 import numpy as np
 import pandas as pd
-from transfers.util import read_csv, filter_to_valid_point_ids, logger, replace_nans
+from transfers.util import read_csv, filter_to_valid_point_ids, replace_nans
+from transfers.logger import logger
 from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address
-
 from schemas.contact import CreateContact, CreateAddress
 
 
@@ -58,9 +58,9 @@ def transfer_contacts(session):
     odf = filter_to_valid_point_ids(session, odf)
     for i, row in odf.iterrows():
         thing = session.query(Thing).where(Thing.name == row.PointID).first()
-        print(f"Processing PointID: {i} {row.PointID}")
+        logger.info(f"Processing PointID: {i} {row.PointID}")
         if thing is None:
-            logger.warning(
+            logger.critical(
                 f"Thing with PointID {row.PointID} not found. Skipping owner."
             )
             continue
@@ -71,7 +71,7 @@ def transfer_contacts(session):
             session.flush()
             logger.info(f"added first contact for PointID {row.PointID}")
         except Exception as e:
-            logger.warning(
+            logger.critical(
                 f"Skipping first contact for PointID {row.PointID} due to validation error: {e}"
             )
             from pprint import pprint
@@ -83,9 +83,9 @@ def transfer_contacts(session):
             add_second_contact(session, row, thing)
             session.commit()
             session.flush()
-            print(f"added second contact for PointID {row.PointID}")
+            logger.info(f"added second contact for PointID {row.PointID}")
         except Exception as e:
-            print(
+            logger.critical(
                 f"Skipping second contact for PointID {row.PointID} due to validation error: {e}"
             )
             session.rollback()
@@ -178,7 +178,7 @@ def add_first_contact(session, row, thing):
             CreateAddress.model_validate(address_data)
             contact.addresses.append(Address(**address_data))
         except Exception as e:
-            logger.warning(
+            logger.critical(
                 f"Skipping physical address for first contact {name}. Validation error: {e}"
             )
 

transfers/data/valid_measuring_agency.csv

@@ -0,0 +1,19 @@
+"MeasuringAgency","valid"
+"Bayard",NO
+"Bernalillo Cty",NO
+"BLM",NO
+"NESWCD",NO
+"NMBGMR",Yes
+"NMED",NO
+"NMISC",NO
+"NMOSE",NO
+"NMT","Yes"
+"NPS",Yes
+"OSWCD",NO
+"PVACD",NO
+"SFC",NO
+"SNL",Yes
+"TSWCD",Yes
+"TWDB",NO
+"USFS",Yes
+"USGS",NO