Make orchestrator skip deleting static files

catalyst-cooperative · Feb 28, 2024 · b6e8cca · b6e8cca
1 parent 893d201
commit b6e8cca
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 2 deletions.
diff --git a/dataset_doi.yaml b/dataset_doi.yaml
@@ -49,6 +49,8 @@ ferc60:
 ferc714:
   production_doi: 10.5281/zenodo.4127100
   sandbox_doi: 10.5072/zenodo.3279
+ferceqr:
+  production_doi: 10.5281/zenodo.10086108
 mshamines:
   production_doi: 10.5281/zenodo.7683517
   sandbox_doi: 10.5072/zenodo.3242

diff --git a/src/pudl_archiver/archivers/classes.py b/src/pudl_archiver/archivers/classes.py
@@ -79,6 +79,9 @@ class AbstractDatasetArchiver(ABC):
     resumeable: bool = False
     existing_files: list[str] = []
 
+    #: List of files that should not be deleted under any circumstance
+    files_not_to_delete: list[str] = []
+
     # Configure which generic validation tests to run
     fail_on_missing_files: bool = True
     fail_on_empty_invalid_files: bool = True
@@ -104,6 +107,9 @@ def __init__(
         """
         self.session = session
 
+        # Never delete datapackage
+        self.files_not_to_delete.append("datapackage.json")
+
         # Create a temporary directory for downloading data
 
         if download_directory is None:

diff --git a/src/pudl_archiver/archivers/ferc/ferceqr.py b/src/pudl_archiver/archivers/ferc/ferceqr.py
@@ -15,9 +15,24 @@ class FercEQRArchiver(AbstractDatasetArchiver):
     """FERC EQR archiver."""
 
     name = "ferceqr"
-    concurrency_limit = 5
+    concurrency_limit = 1
     directory_per_resource_chunk = True
     max_wait_time = 36000
+    files_not_to_delete = [
+        "eqr_nontransaction.zip",
+        "ferceqr-2002.zip",
+        "ferceqr-2003.zip",
+        "ferceqr-2004.zip",
+        "ferceqr-2005.zip",
+        "ferceqr-2006.zip",
+        "ferceqr-2007.zip",
+        "ferceqr-2008.zip",
+        "ferceqr-2009.zip",
+        "ferceqr-2010.zip",
+        "ferceqr-2011.zip",
+        "ferceqr-2012.zip",
+        "ferceqr-2013.zip",
+    ]
 
     async def get_resources(self) -> ArchiveAwaitable:
         """Download FERC EQR resources."""

diff --git a/src/pudl_archiver/orchestrator.py b/src/pudl_archiver/orchestrator.py
@@ -320,7 +320,10 @@ def _get_deletions(
         # Delete files not included in new deposition
         files_to_delete = []
         for filename in draft.files_map:
-            if filename not in resources and filename != "datapackage.json":
+            if (
+                filename not in resources
+                and filename not in self.downloader.files_not_to_delete
+            ):
                 logger.info(f"Deleting {filename} from deposition.")
                 files_to_delete.append(
                     _DepositionChange(_DepositionAction.DELETE, name=filename)