diff --git a/api/thing.py b/api/thing.py index 907bd8470..b81969430 100644 --- a/api/thing.py +++ b/api/thing.py @@ -32,8 +32,9 @@ editor_dependency, viewer_dependency, ) -from db.thing import Thing, ThingIdLink, WellScreen from db.deployment import Deployment +from db.thing import Thing, ThingIdLink, WellScreen +from schemas.deployment import DeploymentResponse from schemas.thing import ( CreateThingIdLink, CreateWell, @@ -49,9 +50,9 @@ UpdateThingIdLink, UpdateWellScreen, ) -from schemas.deployment import DeploymentResponse from services.crud_helper import model_patcher, model_adder, model_deleter from services.exceptions_helper import PydanticStyleException +from services.lexicon_helper import get_terms_by_category from services.query_helper import ( simple_get_by_id, paginated_all_getter, @@ -66,7 +67,6 @@ modify_well_descriptor_tables, WELL_DESCRIPTOR_MODEL_MAP, ) -from services.lexicon_helper import get_terms_by_category router = APIRouter(prefix="/thing", tags=["thing"]) diff --git a/core/initializers.py b/core/initializers.py index 3d43e5787..dbeb2fa23 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -17,6 +17,7 @@ from sqlalchemy import text from sqlalchemy.exc import DatabaseError +from sqlalchemy.orm import Session from db import Base from db.engine import engine, session_ctx @@ -81,6 +82,19 @@ def init_parameter(path: str = None) -> None: session.rollback() +def erase_and_rebuild_db(session: Session): + from sqlalchemy import text + + with session.bind.connect() as conn: + conn.execute(text("DROP SCHEMA public CASCADE")) + conn.execute(text("CREATE SCHEMA public")) + conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) + conn.commit() + + Base.metadata.drop_all(session.bind) + Base.metadata.create_all(session.bind) + + def init_lexicon(path: str = None) -> None: if path is None: path = Path(__file__).parent / "lexicon.json" diff --git a/db/contact.py b/db/contact.py index db0dfce1f..5a4c0f4bb 100644 --- a/db/contact.py +++ b/db/contact.py @@ -13,15 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from typing import List, TYPE_CHECKING + from sqlalchemy import Integer, ForeignKey, String from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, Mapped, mapped_column from sqlalchemy_utils import TSVectorType -from typing import List, TYPE_CHECKING from db.base import Base, AutoBaseMixin, ReleaseMixin, lexicon_term - if TYPE_CHECKING: from db.field import FieldEventParticipant, FieldEvent from db.thing import Thing @@ -118,7 +118,8 @@ class Phone(Base, AutoBaseMixin, ReleaseMixin): contact_id: Mapped[int] = mapped_column( ForeignKey("contact.id", ondelete="CASCADE"), nullable=False ) - phone_number: Mapped[str] = mapped_column(String(20), nullable=False) + nma_phone_number: Mapped[str] = mapped_column(String(20), nullable=True) + phone_number: Mapped[str] = mapped_column(String(20), nullable=True) phone_type: Mapped[str] = lexicon_term(nullable=False) contact: Mapped["Contact"] = relationship( diff --git a/db/geochronology.py b/db/geochronology.py index 18cf1dab7..9b2796cfc 100644 --- a/db/geochronology.py +++ b/db/geochronology.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from db.base import AutoBaseMixin, Base, lexicon_term from sqlalchemy import Integer, Float from sqlalchemy.orm import mapped_column +from db.base import AutoBaseMixin, Base, lexicon_term + class GeochronologyAge(Base, AutoBaseMixin): location_id = mapped_column(Integer, nullable=False) diff --git a/schemas/__init__.py b/schemas/__init__.py index c18cec53e..e1c81b75e 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== from datetime import datetime, timezone + from pydantic import BaseModel, ConfigDict, AwareDatetime from pydantic.json_schema import JsonSchemaValue from pydantic_core import core_schema diff --git a/schemas/contact.py b/schemas/contact.py index 7fde8f24c..07bbf4cff 100644 --- a/schemas/contact.py +++ b/schemas/contact.py @@ -67,15 +67,26 @@ def validate_phone(cls, phone_number_str: str | None) -> str | None: region = "US" try: phone_number_str = phone_number_str.strip() + parsed_number = phonenumbers.parse(phone_number_str, region) + if phonenumbers.is_valid_number(parsed_number): + formatted_number = phonenumbers.format_number( + parsed_number, phonenumbers.PhoneNumberFormat.E164 + ) + return formatted_number + # this is a major hack to deal with the phone numbers entered into # NM_Aquifer without an area code - for p in (phone_number_str, f"505{phone_number_str}"): - parsed_number = phonenumbers.parse(p, region) - if phonenumbers.is_valid_number(parsed_number): - formatted_number = phonenumbers.format_number( - parsed_number, phonenumbers.PhoneNumberFormat.E164 - ) - return formatted_number + # for p in ( + # phone_number_str, + # f"505{phone_number_str}", + # f"575{phone_number_str}", + # ): + # parsed_number = phonenumbers.parse(p, region) + # if phonenumbers.is_valid_number(parsed_number): + # formatted_number = phonenumbers.format_number( + # parsed_number, phonenumbers.PhoneNumberFormat.E164 + # ) + # return formatted_number else: raise ValueError(f"Invalid phone number. {phone_number_str}") except NumberParseException as e: @@ -89,7 +100,6 @@ class CreateEmail(BaseCreateModel, ValidateEmail): """ contact_id: int | None = None # set to None for when made via POST /contact - email: str email_type: str = "Primary" # Default to 'Primary' @@ -99,8 +109,8 @@ class CreatePhone(BaseCreateModel, ValidatePhone): """ contact_id: int | None = None # set to None for when made via POST /contact - phone_number: str phone_type: str = "Primary" # Default to 'Primary' + nma_phone_number: str | None = None class CreateAddress(BaseCreateModel): @@ -160,8 +170,9 @@ class PhoneResponse(BaseItemResponse): Response schema for phone details. """ - phone_number: str + phone_number: str | None = None phone_type: str # e.g., 'mobile', 'landline', etc. + nma_phone_number: str | None = None class EmailResponse(BaseItemResponse): diff --git a/schemas/lexicon.py b/schemas/lexicon.py index 00f624378..01519feff 100644 --- a/schemas/lexicon.py +++ b/schemas/lexicon.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from pydantic import BaseModel, ConfigDict from typing import List + +from pydantic import BaseModel, ConfigDict + from schemas import UTCAwareDatetime diff --git a/schemas/observation.py b/schemas/observation.py index c8a06994f..39296dfea 100644 --- a/schemas/observation.py +++ b/schemas/observation.py @@ -14,6 +14,8 @@ # limitations under the License. # =============================================================================== from datetime import timezone +from typing import Annotated + from pydantic import ( BaseModel, AwareDatetime, @@ -21,7 +23,6 @@ field_validator, model_validator, ) -from typing import Annotated from typing_extensions import Self from schemas import ( diff --git a/schemas/sample.py b/schemas/sample.py index b928af7e4..bad0682ea 100644 --- a/schemas/sample.py +++ b/schemas/sample.py @@ -14,6 +14,8 @@ # limitations under the License. # =============================================================================== from datetime import timezone +from typing import Annotated + from pydantic import ( BaseModel, field_validator, @@ -21,7 +23,6 @@ AwareDatetime, PastDatetime, ) -from typing import Annotated from typing_extensions import Self from schemas import ( @@ -30,9 +31,9 @@ BaseResponseModel, UTCAwareDatetime, ) -from schemas.thing import ThingResponse -from schemas.field import FieldEventResponse, FieldActivityResponse from schemas.contact import ContactResponse +from schemas.field import FieldEventResponse, FieldActivityResponse +from schemas.thing import ThingResponse # -------- VALIDATE ---------- diff --git a/schemas/thing.py b/schemas/thing.py index 21704e9fa..f40dedf28 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -20,6 +20,7 @@ from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel from schemas.location import LocationResponse + # -------- VALIDATE ---------- diff --git a/tests/__init__.py b/tests/__init__.py index f341d6c89..8651b6391 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -30,24 +30,14 @@ time.tzset() from fastapi.testclient import TestClient -from sqlalchemy import text -from core.initializers import init_lexicon, init_parameter +from core.initializers import init_lexicon, init_parameter, erase_and_rebuild_db from db import Base, Parameter -from db.engine import engine, session_ctx +from db.engine import session_ctx from main import app - -# Force clean database state by dropping and recreating schema -# This ensures test isolation similar to Docker environment -with engine.connect() as conn: - # Drop all tables with CASCADE to handle foreign key dependencies - conn.execute(text("DROP SCHEMA IF EXISTS public CASCADE")) - conn.execute(text("CREATE SCHEMA public")) - conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) - conn.commit() - -Base.metadata.create_all(engine) +with session_ctx() as session: + erase_and_rebuild_db(session) init_lexicon() init_parameter() diff --git a/tests/test_asset.py b/tests/test_asset.py index 1e0859843..1502075c1 100644 --- a/tests/test_asset.py +++ b/tests/test_asset.py @@ -13,15 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from datetime import timezone +from unittest.mock import patch + +import pytest + from api.asset import get_storage_bucket from core.app import app from core.dependencies import viewer_function, admin_function, editor_function from db import Asset from tests import client, cleanup_post_test, override_authentication, cleanup_patch_test -import pytest -from datetime import timezone -from unittest.mock import patch # CLASSES, FIXTURES, AND FUNCTIONS ============================================= diff --git a/tests/test_contact.py b/tests/test_contact.py index bfa2c8781..37890a918 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -1,3 +1,9 @@ +import re +from datetime import timezone + +import pytest +from pydantic import ValidationError + from core.dependencies import ( amp_viewer_function, amp_editor_function, @@ -5,13 +11,8 @@ ) from db import Contact, Address, Email, Phone from main import app -from tests import client, cleanup_post_test, cleanup_patch_test, override_authentication from schemas.contact import ValidateEmail, ValidatePhone, ValidateContact - -import pytest -from datetime import timezone -from pydantic import ValidationError -import re +from tests import client, cleanup_post_test, cleanup_patch_test, override_authentication @pytest.fixture(scope="module", autouse=True) diff --git a/tests/test_group.py b/tests/test_group.py index 1591a7255..41a312621 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -1,10 +1,11 @@ +from datetime import timezone + +import pytest from geoalchemy2.shape import to_shape from pydantic import ValidationError -import pytest -from datetime import timezone -from db import Group from core.dependencies import admin_function, viewer_function, editor_function +from db import Group from main import app from schemas.group import ValidateGroup from tests import client, override_authentication, cleanup_post_test, cleanup_patch_test diff --git a/tests/test_lexicon.py b/tests/test_lexicon.py index cd2fca0a9..c52550ee9 100644 --- a/tests/test_lexicon.py +++ b/tests/test_lexicon.py @@ -13,18 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from db import LexiconTerm, LexiconCategory, LexiconTriple -from tests import client, override_authentication, cleanup_post_test, cleanup_patch_test +from datetime import timezone + +import pytest from core.dependencies import ( viewer_function, lexicon_admin_function, lexicon_editor_function, ) +from db import LexiconTerm, LexiconCategory, LexiconTriple from main import app - -import pytest -from datetime import timezone +from tests import client, override_authentication, cleanup_post_test, cleanup_patch_test @pytest.fixture(scope="module", autouse=True) diff --git a/tests/test_location.py b/tests/test_location.py index 8feff3a99..5c4bbf5e0 100644 --- a/tests/test_location.py +++ b/tests/test_location.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from geoalchemy2.shape import to_shape -import pytest from datetime import timezone +import pytest +from geoalchemy2.shape import to_shape + from core.dependencies import admin_function, editor_function, viewer_function from db import Location from main import app diff --git a/tests/test_observation.py b/tests/test_observation.py index ce9aa1ad0..5bdecff3f 100644 --- a/tests/test_observation.py +++ b/tests/test_observation.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -import pytest from datetime import timezone -from db import Observation +import pytest + from core.dependencies import ( amp_admin_function, admin_function, @@ -24,6 +24,7 @@ amp_editor_function, viewer_function, ) +from db import Observation from main import app from tests import ( client, diff --git a/tests/test_sample.py b/tests/test_sample.py index 6c817f1fe..5fc361615 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -13,13 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -import pytest from datetime import timezone + +import pytest from pydantic import ValidationError -from main import app from core.dependencies import admin_function, editor_function, viewer_function from db.sample import Sample +from main import app from schemas.sample import ValidateSample from tests import client, cleanup_post_test, cleanup_patch_test, override_authentication diff --git a/tests/test_sensor.py b/tests/test_sensor.py index 294cda201..7b0aff033 100644 --- a/tests/test_sensor.py +++ b/tests/test_sensor.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from datetime import timezone + +import pytest + from core.dependencies import admin_function, editor_function, viewer_function from db import Sensor from main import app @@ -26,8 +30,6 @@ groundwater_level_parameter_id, ) -import pytest -from datetime import timezone # from pydantic import ValidationError diff --git a/tests/test_thing.py b/tests/test_thing.py index 82d0c9d45..bf64a5f94 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -import pytest from datetime import timezone -from db import Thing, WellScreen, ThingIdLink -from tests import client, override_authentication, cleanup_post_test, cleanup_patch_test -from main import app +import pytest + from core.dependencies import ( admin_function, editor_function, @@ -27,8 +25,11 @@ viewer_function, amp_viewer_function, ) +from db import Thing, WellScreen, ThingIdLink +from main import app from schemas.location import LocationResponse from schemas.thing import ValidateWell +from tests import client, override_authentication, cleanup_post_test, cleanup_patch_test @pytest.fixture(scope="module", autouse=True) diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index 52c082a42..d2eba663f 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -15,10 +15,10 @@ # =============================================================================== from pydantic import ValidationError -from transfers.util import read_csv, filter_to_valid_point_ids, replace_nans -from transfers.logger import logger from db import Thing, Contact, ThingContactAssociation, Email, Phone, Address from schemas.contact import CreateContact, CreateAddress, CreatePhone, CreateEmail +from transfers.logger import logger +from transfers.util import read_csv, filter_to_valid_point_ids, replace_nans def extract_owner_role(comment): @@ -258,12 +258,21 @@ def _make_phone(first_second, ownerkey, **kw): try: if "phone_number" in kw: kw["phone_number"] = kw["phone_number"].strip() + phone = CreatePhone(**kw) return Phone(**phone.model_dump()) except ValidationError as e: - logger.critical( - f"{first_second} '{ownerkey}' Skipping phone . Validation error: {e.errors()}" - ) + try: + if "phone_number" in kw: + pn = kw.pop("phone_number") + kw["nma_phone_number"] = pn.strip() + phone = CreatePhone(**kw) + return Phone(**phone.model_dump()) + except ValidationError: + + logger.critical( + f"{first_second} '{ownerkey}' Skipping phone . Validation error: {e.errors()}" + ) def _make_address(first_second, ownerkey, kind, **kw): diff --git a/transfers/logger.py b/transfers/logger.py index aab12079b..2c9720c50 100644 --- a/transfers/logger.py +++ b/transfers/logger.py @@ -18,7 +18,6 @@ from services.gcs_helper import get_storage_bucket - # class StreamToLogger: # def __init__(self, logger_, level): # self.logger = logger_ diff --git a/transfers/sensor_transfer.py b/transfers/sensor_transfer.py index 86635643a..df80ba1f6 100644 --- a/transfers/sensor_transfer.py +++ b/transfers/sensor_transfer.py @@ -14,10 +14,11 @@ # limitations under the License. # =============================================================================== from datetime import datetime + import pandas as pd from db import Sensor, Deployment, Thing -from transfers.util import read_csv, logger +from transfers.util import read_csv, logger, filter_to_valid_point_ids, replace_nans EQUIPMENT_TO_SENSOR_TYPE_MAP = { "Pressure transducer": "Pressure Transducer", @@ -29,8 +30,11 @@ def transfer_sensors(session): equipment = read_csv("Equipment") equipment.columns = equipment.columns.str.replace(" ", "_") - grouped_equipment = equipment.groupby(["PointID"]) + equipment = equipment[equipment.SerialNo.notna()] + equipment = filter_to_valid_point_ids(session, equipment) + equipment = replace_nans(equipment) + grouped_equipment = equipment.groupby(["PointID"]) for index, group in grouped_equipment: pointid = index[0] thing = session.query(Thing).filter(Thing.name == pointid).first() @@ -41,9 +45,6 @@ def transfer_sensors(session): continue ordered_group = group.sort_values(by=["DateInstalled"]) - if pointid == "SO-0168": - print(ordered_group) - try: for row in ordered_group.itertuples(): if row.EquipmentType not in EQUIPMENT_TO_SENSOR_TYPE_MAP: diff --git a/transfers/transfer.py b/transfers/transfer.py index ecef2a361..748213fbe 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -19,11 +19,8 @@ load_dotenv() - -from sqlalchemy import text from sqlalchemy.orm import Session -from core.initializers import init_lexicon, init_parameter -from db import Base +from core.initializers import init_lexicon, init_parameter, erase_and_rebuild_db from db.engine import session_ctx from transfers.group_transfer import transfer_groups @@ -71,16 +68,8 @@ def parameter(): @timeit def erase(session: Session): - logger.info("Erasing existing data") - with session.bind.connect() as conn: - conn.execute(text("DROP SCHEMA public CASCADE")) - conn.execute(text("CREATE SCHEMA public")) - conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) - conn.commit() - - Base.metadata.drop_all(session.bind) - logger.info("Recreating tables") - Base.metadata.create_all(session.bind) + logger.info("Erase and rebuilding database") + erase_and_rebuild_db(session) def message(msg, pad=10, new_line_at_top=True): @@ -99,6 +88,9 @@ def transfer_all(sess, limit=100): timeit_direct(transfer_wells, sess, limit=limit) timeit_direct(transfer_wellscreens, sess) + message("TRANSFERRING SENSORS") + timeit_direct(transfer_sensors, sess) + """ Developer's note this is a very time consuming operation and the results should @@ -117,9 +109,6 @@ def transfer_all(sess, limit=100): message("TRANSFERRING METEOROLOGICAL") timeit_direct(transfer_met, sess, limit) - message("TRANSFERRING SENSORS") - timeit_direct(transfer_sensors, sess) - message("TRANSFERRING CONTACTS") timeit_direct(transfer_contacts, sess) @@ -152,7 +141,7 @@ def transfer_all(sess, limit=100): def main(): message("START--------------------------------------") - limit = int(os.environ.get("TRANSFER_LIMIT", 10000)) + limit = int(os.environ.get("TRANSFER_LIMIT", 1000)) with session_ctx() as sess: transfer_all(sess, limit=limit) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index 916801c06..0c26ee783 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -15,10 +15,11 @@ # =============================================================================== import json import time -from pydantic import ValidationError -from sqlalchemy import select from datetime import datetime + from pandas import isna +from pydantic import ValidationError +from sqlalchemy import select from db import ( LocationThingAssociation, @@ -155,7 +156,6 @@ def transfer_wells(session, limit=0) -> None: # so that effective_start can be set on the location assocation data = CreateWell( location_id=location.id, - nma_pk_welldata=row.WellID, name=row.PointID, first_visit_date=first_visit_date, hole_depth=row.HoleDepth, @@ -184,6 +184,7 @@ def transfer_wells(session, limit=0) -> None: ] ) well_data["thing_type"] = "water well" + well_data["nma_pk_welldata"] = row.WellID well = Thing(**well_data) session.add(well) diff --git a/uv.lock b/uv.lock index 332a2e60b..ff0c59d03 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.13" [[package]]