Merge branch 'dev' into python-3.11

docker/.env

@@ -6,6 +6,6 @@ PUDL_OUTPUT=/home/catalyst/pudl_work/output
 DAGSTER_HOME=/home/catalyst/pudl_work/dagster_home
 CONDA_PREFIX=/home/catalyst/env
 PUDL_SETTINGS_YML=/home/catalyst/src/pudl/package_data/settings/etl_full.yml
-LOGFILE=/home/catalyst/pudl_out/pudl-etl.log
+LOGFILE=/home/catalyst/pudl_work/output/pudl-etl.log
 CONDA_RUN="conda run --no-capture-output --prefix /home/catalyst/env"
 GCS_CACHE=gs://zenodo-cache.catalyst.coop

docker/gcp_pudl_etl.sh

@@ -42,7 +42,7 @@ function shutdown_vm() {
     # Copy the outputs to the GCS bucket
     gsutil -m cp -r $PUDL_OUTPUT "gs://nightly-build-outputs.catalyst.coop/$ACTION_SHA-$GITHUB_REF"
 
-    upload_file_to_slack "${PUDL_OUTPUT}/pudl-etl.log" "Logs for $ACTION_SHA-$GITHUB_REF:"
+    upload_file_to_slack $LOGFILE "pudl_etl logs for $ACTION_SHA-$GITHUB_REF:"
 
     echo "Shutting down VM."
     # # Shut down the vm instance when the etl is done.

setup.py

@@ -48,7 +48,6 @@
     setup_requires=["setuptools_scm"],
     install_requires=[
         "addfips>=0.4,<0.5",
-        "alembic>=1.9.4,<2",
         "catalystcoop.dbfread>=3.0,<3.1",
         "coloredlogs>=14.0,<15.1",  # Dagster requires 14.0
         "catalystcoop.ferc-xbrl-extractor==0.8.1",

src/pudl/io_managers.py

@@ -7,8 +7,6 @@
 import pandas as pd
 import pyarrow as pa
 import sqlalchemy as sa
-from alembic.autogenerate import compare_metadata
-from alembic.migration import MigrationContext
 from dagster import (
     Field,
     InitResourceContext,
@@ -47,7 +45,7 @@ def __str__(self):
         """Create string representation of ForeignKeyError object."""
         return (
             f"Foreign key error for table: {self.child_table} -- {self.parent_table} "
-            "{self.foreign_key} -- on rows {self.rowids}\n"
+            f"{self.foreign_key} -- on rows {self.rowids}\n"
         )
 
     def __eq__(self, other):
@@ -83,12 +81,6 @@ def __getitem__(self, idx):
         return self.fk_errors[idx]
 
 
-class MetadataDiffError(ValueError):
-    """Raised when a sqlalchemy metadata object differs from a databases schema."""
-
-    pass
-
-
 class SQLiteIOManager(IOManager):
     """IO Manager that writes and retrieves dataframes from a SQLite database."""
 
@@ -126,9 +118,9 @@ def __init__(
             )
 
         # If no metadata is specified, create an empty sqlalchemy metadata object.
-        if md is None:
-            md = sa.MetaData()
         self.md = md
+        if not self.md:
+            self.md = sa.MetaData()
 
         self.engine = self._setup_database(timeout=timeout)
 
@@ -165,17 +157,6 @@ def _setup_database(self, timeout: float = 1_000.0) -> sa.engine.Engine:
         if not db_path.exists():
             db_path.touch()
             self.md.create_all(engine)
-        else:
-            # Compare the new metadata with the existing metadata in the db.
-            # If they are different, raise an error to clobber the database
-            mc = MigrationContext.configure(engine.connect())
-            metadata_diff = compare_metadata(mc, self.md)
-            if metadata_diff:
-                logger.info("Metadata diff:\n\n{metadata_diff}")
-                raise MetadataDiffError(
-                    "The database schema has changed. Delete the "
-                    f"database at {db_path}"
-                )
 
         return engine
 

test/unit/io_managers_test.py

@@ -5,17 +5,12 @@
 from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table
 from sqlalchemy.exc import IntegrityError, OperationalError
 
-from pudl.io_managers import (
-    ForeignKeyError,
-    ForeignKeyErrors,
-    MetadataDiffError,
-    SQLiteIOManager,
-)
+from pudl.io_managers import ForeignKeyError, ForeignKeyErrors, SQLiteIOManager
 
 
 @pytest.fixture
-def db_metadata() -> MetaData:
-    """Create a sample metadata fixture for io manager tests."""
+def sqlite_io_manager_fixture(tmp_path):
+    """Create a SQLiteIOManager fixture with a simple database schema."""
     md = MetaData()
     artist = Table(  # noqa: F841
         "artist",
@@ -30,50 +25,8 @@ def db_metadata() -> MetaData:
         Column("trackname", String(16), nullable=False),
         Column("trackartist", Integer, ForeignKey("artist.artistid")),
     )
-    return md
-
-
-@pytest.fixture
-def sqlite_io_manager_fixture(tmp_path, db_metadata):
-    """Create a SQLiteIOManager fixture with a simple database schema."""
-    return SQLiteIOManager(base_dir=tmp_path, db_name="pudl", md=db_metadata)
-
-
-def test_metadata_change_error(sqlite_io_manager_fixture):
-    """Test a MetadataDiffError is raised when the metadata changes."""
-    md = MetaData()
-    artist = Table(  # noqa: F841
-        "artist",
-        md,
-        Column("artistid", Integer, primary_key=True),
-        Column("artistname", String(16), nullable=False),
-    )
-    base_dir = sqlite_io_manager_fixture.base_dir
-
-    with pytest.raises(MetadataDiffError):
-        SQLiteIOManager(base_dir=base_dir, db_name="pudl", md=md)
-
-
-def test_unchanged_metadata(sqlite_io_manager_fixture, db_metadata):
-    """Test a MetadataDiffError isn't raised when the metadata doesn't change."""
-    base_dir = sqlite_io_manager_fixture.base_dir
-
-    SQLiteIOManager(base_dir=base_dir, db_name="pudl", md=db_metadata)
-
-
-def test_unchanged_metadata_with_view(sqlite_io_manager_fixture, db_metadata):
-    """Test a MetadataDiffError isn't raised when a view is created."""
-    query = """CREATE VIEW tracks_view AS SELECT * FROM track;"""
-
-    engine = sqlite_io_manager_fixture.engine
-    with engine.connect() as conn:
-        conn.execute(query)
-        tracks_view = pd.read_sql_table("tracks_view", conn)
-    assert tracks_view.shape == (0, 3)
-
-    base_dir = sqlite_io_manager_fixture.base_dir
 
-    SQLiteIOManager(base_dir=base_dir, db_name="pudl", md=db_metadata)
+    return SQLiteIOManager(base_dir=tmp_path, db_name="pudl", md=md)
 
 
 def test_sqlite_io_manager_delete_stmt(sqlite_io_manager_fixture):