Add set up function for pipelines. (#912)

mirar/pipelines/base_pipeline.py

@@ -73,6 +73,7 @@ def __init__(
             selected_configurations = [selected_configurations]
         self.selected_configurations = selected_configurations
         self.latest_configuration = None
+        self.set_up_pipeline()
 
     @classmethod
     def __init_subclass__(cls, **kwargs):
@@ -87,6 +88,13 @@ def __init_subclass__(cls, **kwargs):
             raise ValueError(err)
         cls.pipelines[cls.name] = cls
 
+    def set_up_pipeline(self):
+        """
+        Function to do any additional pipeline setup.
+
+        :return: None
+        """
+
     def load_pipeline_configuration(
         self,
         configuration: str = "default",

mirar/pipelines/summer/models/__init__.py

@@ -3,6 +3,8 @@
 """
 
 # pylint: disable=duplicate-code
+import logging
+
 from mirar.database.credentials import DB_USER
 from mirar.database.setup import setup_database
 from mirar.pipelines.summer.models._diff import Diff, DiffTable
@@ -47,10 +49,19 @@
 )
 from mirar.pipelines.summer.models.base_model import SummerBase
 
-if DB_USER is not None:
-    setup_database(SummerBase)
-    populate_fields()
-    populate_itid()
-    populate_filters()
-    populate_programs()
-    populate_subdets()
+logger = logging.getLogger(__name__)
+
+
+def set_up_summer_databases():
+    """
+    Function to set up the summer databases
+    """
+    if DB_USER is not None:
+        setup_database(SummerBase)
+        populate_fields()
+        populate_itid()
+        populate_filters()
+        populate_programs()
+        populate_subdets()
+    else:
+        logging.warning("DB_USER not set, skipping SUMMER database setup")

mirar/pipelines/summer/summer_pipeline.py

@@ -8,7 +8,6 @@
 
 from mirar.data import Image
 from mirar.downloader.caltech import download_via_ssh
-from mirar.io import open_raw_image
 from mirar.pipelines.base_pipeline import Pipeline
 from mirar.pipelines.summer.blocks import (
     build_log,
@@ -26,6 +25,7 @@
 )
 from mirar.pipelines.summer.config import PIPELINE_NAME, summer_cal_requirements
 from mirar.pipelines.summer.load_summer_image import load_raw_summer_image
+from mirar.pipelines.summer.models import set_up_summer_databases
 
 summer_flats_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)))
 
@@ -69,3 +69,6 @@ def download_raw_images_for_night(night: str | int):
     @staticmethod
     def _load_raw_image(path: str | Path) -> Image | list[Image]:
         return load_raw_summer_image(path)
+
+    def set_up_pipeline(self):
+        set_up_summer_databases()

mirar/pipelines/winter/models/__init__.py

@@ -84,21 +84,31 @@ def set_up_q3c(db_name: str, db_table: BaseTable):
     )
 
 
-if DB_USER is not None:
-    setup_database(db_base=WinterBase)
+def set_up_winter_databases():
+    """
+    Setup the winter databases
+
+    :return: None
+    """
+
+    if DB_USER is not None:
+        setup_database(db_base=WinterBase)
+
+        for table in [
+            ExposuresTable,
+            AstrometryStatsTable,
+            CandidatesTable,
+            RefQueriesTable,
+            StacksTable,
+            SourcesTable,
+        ]:
+            set_up_q3c(db_name=WinterBase.db_name, db_table=table)
 
-    for table in [
-        ExposuresTable,
-        AstrometryStatsTable,
-        CandidatesTable,
-        RefQueriesTable,
-        StacksTable,
-        SourcesTable,
-    ]:
-        set_up_q3c(db_name=WinterBase.db_name, db_table=table)
+        populate_fields()
+        populate_itid()
+        populate_filters()
+        populate_programs()
+        populate_subdets()
 
-    populate_fields()
-    populate_itid()
-    populate_filters()
-    populate_programs()
-    populate_subdets()
+    else:
+        logger.warning("No database user provided. Skipping WINTER database setup.")

mirar/pipelines/winter/winter_pipeline.py

@@ -55,6 +55,7 @@
 )
 from mirar.pipelines.winter.config import PIPELINE_NAME, winter_cal_requirements
 from mirar.pipelines.winter.load_winter_image import load_raw_winter_mef
+from mirar.pipelines.winter.models import set_up_winter_databases
 
 logger = logging.getLogger(__name__)
 
@@ -154,3 +155,6 @@ def download_raw_images_for_night(night: str):
             pipeline=PIPELINE_NAME,
             server_sub_dir="raw",
         )
+
+    def set_up_pipeline(self):
+        set_up_winter_databases()

mirar/pipelines/wirc/wirc_files/models.py

@@ -146,5 +146,11 @@ class Candidate(BaseDB):
     isdiffpos: bool = Field(default=True)
 
 
-if DB_USER is not None:
-    setup_database(WircBase)
+def set_up_wirc_database():
+    """
+    Function to set up the wirc database
+    """
+    if DB_USER is not None:
+        setup_database(WircBase)
+    else:
+        logger.warning("No database user set, skipping WIRC database setup")

mirar/pipelines/wirc/wirc_pipeline.py

@@ -10,6 +10,7 @@
 from mirar.pipelines.base_pipeline import Pipeline
 from mirar.pipelines.wirc.blocks import imsub, load_raw, reduce
 from mirar.pipelines.wirc.load_wirc_image import load_raw_wirc_image
+from mirar.pipelines.wirc.wirc_files.models import set_up_wirc_database
 
 logger = logging.getLogger(__name__)
 
@@ -45,3 +46,6 @@ def download_raw_images_for_night(night: str | int):
     @staticmethod
     def _load_raw_image(path: str | Path) -> Image | list[Image]:
         return load_raw_wirc_image(path)
+
+    def set_up_pipeline(self):
+        set_up_wirc_database()