Merge pull request #178 from pepkit/dev_record_history

Create History for Filebackend and DB Backend

.github/workflows/cli-coverage.yml

@@ -3,6 +3,8 @@ name: Pipestat Test Coverage
 on:
   push:
     branches: [master, dev]
+  pull_request:
+    branches: [ dev ]
 
 jobs:
   cli-coverage-report:
@@ -18,7 +20,7 @@ jobs:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: pipestat-password
           POSTGRES_DB: pipestat-test
-          #POSTGRES_HOST: localhost
+          POSTGRES_HOST: localhost
         ports:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
@@ -56,4 +58,4 @@ jobs:
           SMOKESHOW_GITHUB_CONTEXT: coverage
           SMOKESHOW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SMOKESHOW_GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-          SMOKESHOW_AUTH_KEY: ${{ secrets.SMOKESHOW_AUTH_KEY }}
+          SMOKESHOW_AUTH_KEY: ${{ secrets.SMOKESHOW_AUTH_KEY }}

.github/workflows/run-pytest.yml

@@ -48,8 +48,8 @@ jobs:
     - name: Run pytest tests
       run: pytest tests -x -vv --cov=./ --cov-report=xml
 
-    - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v1
-      with:
-        file: ./coverage.xml
-        name: py-${{ matrix.python-version }}-${{ matrix.os }}
+#    - name: Upload coverage to Codecov
+#      uses: codecov/codecov-action@v1
+#      with:
+#        file: ./coverage.xml
+#        name: py-${{ matrix.python-version }}-${{ matrix.os }}

docs/changelog.md

@@ -2,9 +2,19 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
-## [0.8.3] - 2024-03-xx
+## [0.9.0] - 2024-04-xx
 ### Fixed
 - Bug with rm_record for filebackend
+### Added
+- Added results history and history retrieval for both file and db backends via `retrieve_history` [#177](https://github.com/pepkit/pipestat/issues/177).
+- Added `remove_record` to Pipestat manager object (it is no longer only on backend classes)
+- Added `meta` key to each record for the file backend
+- db backend will now create an additional sql history table
+- Reporting history is toggleable
+### Changed
+- Removing the last result no longer removes the entire record.
+- `pipestat_created_time, and `pipestat_modified_time` now live under the `meta` key.
+- `history` lives under the `meta` key for the filebackend.
 
 ## [0.8.2] - 2024-02-22
 ### Changed

pipestat/argparser.py

@@ -11,6 +11,7 @@
 INSPECT_CMD = "inspect"
 REMOVE_CMD = "remove"
 RETRIEVE_CMD = "retrieve"
+HISTORY_CMD = "history"
 STATUS_CMD = "status"
 INIT_CMD = "init"
 SUMMARIZE_CMD = "summarize"
@@ -26,6 +27,7 @@
     SUMMARIZE_CMD: "Generates HTML Report",
     LINK_CMD: "Create symlinks of reported files",
     SERVE_CMD: "Initializes pipestatreader API",
+    HISTORY_CMD: "Retrieve history of reported results for one record identifier",
 }
 
 STATUS_GET_CMD = "get"
@@ -169,7 +171,7 @@ def add_subparser(
         )
 
     # remove, report and inspect
-    for cmd in [REMOVE_CMD, REPORT_CMD, INSPECT_CMD, RETRIEVE_CMD]:
+    for cmd in [REMOVE_CMD, REPORT_CMD, INSPECT_CMD, RETRIEVE_CMD, HISTORY_CMD]:
         sps[cmd].add_argument(
             "-f",
             "--results-file",
@@ -239,13 +241,21 @@ def add_subparser(
             help=f"ID of the record to report the result for. {_env_txt('record_identifier')}",
         )
 
-    sps[RETRIEVE_CMD].add_argument(
-        "-r",
-        "--record-identifier",
-        type=str,
-        metavar="R",
-        help=f"ID of the record to report the result for. {_env_txt('record_identifier')}",
-    )
+    for cmd in [RETRIEVE_CMD, HISTORY_CMD]:
+        sps[cmd].add_argument(
+            "-i",
+            "--result-identifier",
+            type=str,
+            metavar="I",
+            help="ID of the result to report; needs to be defined in the schema",
+        )
+        sps[cmd].add_argument(
+            "-r",
+            "--record-identifier",
+            type=str,
+            metavar="R",
+            help=f"ID of the record to report the result for. {_env_txt('record_identifier')}",
+        )
 
     # report
     sps[REPORT_CMD].add_argument(

pipestat/backends/abstract.py

@@ -153,6 +153,7 @@ def report(
         record_identifier: str,
         force_overwrite: bool = False,
         result_formatter: Optional[staticmethod] = None,
+        history_enabled: bool = True,
     ) -> str:
         _LOGGER.warning("Not implemented yet for this backend")
 

pipestat/backends/db_backend/db_parsed_schema.py

@@ -196,6 +196,49 @@ def _get_data_type(type_name):
     def file_like_table_name(self):
         return self._table_name("files")
 
+    def build_history_model(self, pipeline_type):
+        """Creates model for history ORM
+        :param str pipeline_type: project or sample-level pipeline
+        :return model: (model, table_name)
+        """
+        if pipeline_type == "project":
+            history_table_name = self.project_table_name + "_history"
+            data = self.project_level_data
+            main_table_id = self.project_table_name + ".id"
+
+        elif pipeline_type == "sample":
+            history_table_name = self.sample_table_name + "_history"
+            data = self.sample_level_data
+            main_table_id = self.sample_table_name + ".id"
+
+        else:
+            raise PipestatError(
+                f"Building model requires pipeline type. Provided type: '{pipeline_type}' "
+            )
+
+        if not self.sample_level_data and not self.project_level_data:
+            return None
+
+        field_defs = self._make_field_definitions(data, require_type=True)
+        field_defs = self._add_status_field(field_defs)
+        field_defs = self._add_record_identifier_field(field_defs)
+        field_defs = self._add_id_field(field_defs)
+        field_defs = self._add_pipeline_name_field(field_defs)
+        field_defs = self._add_created_time_field(field_defs)
+        field_defs = self._add_modified_time_field(field_defs)
+
+        field_defs["source_record_id"] = (
+            int,
+            Field(
+                default=None,
+                foreign_key=main_table_id,
+            ),
+        )
+
+        history_model = _create_model(history_table_name, **field_defs)
+
+        return history_model, history_table_name
+
     def build_model(self, pipeline_type):
         if pipeline_type == "project":
             data = self.project_level_data