Refactor data analysis and pipeline to workflow

alembic/versions/2024_02_09_e907e651fb9e_rename_data_analysis_to_workflow.py

@@ -0,0 +1,29 @@
+"""rename_data_analysis_to_workflow
+
+Revision ID: e907e651fb9e
+Revises: 4b925b29ac45
+Create Date: 2024-02-09 16:45:01.272807
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "e907e651fb9e"
+down_revision = "4b925b29ac45"
+branch_labels = None
+depends_on = None
+
+import sqlalchemy as sa
+
+from alembic import op
+
+
+def upgrade():
+    op.alter_column(
+        "analysis", "data_analysis", new_column_name="workflow", existing_type=sa.String(32)
+    )  # Drop the unique constraint
+
+
+def downgrade():
+    op.alter_column(
+        "analysis", "workflow", new_column_name="data_analysis", existing_type=sa.String(32)
+    )

tests/apps/slurm/test_api.py

@@ -7,7 +7,7 @@
     reformat_squeue_result_job_step,
 )
 from trailblazer.apps.slurm.models import SqueueResult
-from trailblazer.constants import Pipeline, SlurmJobStatus, TrailblazerStatus
+from trailblazer.constants import SlurmJobStatus, TrailblazerStatus, Workflow
 
 
 def test_get_squeue_result(squeue_stream_jobs):
@@ -40,49 +40,47 @@ def test_get_squeue_jobs_flag_input(job_id: dict[str, list[str]], expected_job_i
 
 
 @pytest.mark.parametrize(
-    "data_analysis, raw_job_step, expected_job_step",
+    "workflow, raw_job_step, expected_job_step",
     [
-        (Pipeline.MIP_DNA, "mipdnastep_jobstep", "mipdnastep"),
-        (Pipeline.MIP_RNA, "miprnastep_jobstep", "miprnastep"),
-        (Pipeline.BALSAMIC, "job.step.balsamicstep", "balsamicstep"),
-        (Pipeline.SARS_COV_2, "job_step_mutantstep", "mutantstep"),
+        (Workflow.MIP_DNA, "mipdnastep_jobstep", "mipdnastep"),
+        (Workflow.MIP_RNA, "miprnastep_jobstep", "miprnastep"),
+        (Workflow.BALSAMIC, "job.step.balsamicstep", "balsamicstep"),
+        (Workflow.MUTANT, "job_step_mutantstep", "mutantstep"),
     ],
 )
-def test_reformat_squeue_result_job_step(
-    data_analysis: str, raw_job_step: str, expected_job_step: str
-):
-    """Test reformatting job step name for each pipeline."""
+def test_reformat_squeue_result_job_step(workflow: str, raw_job_step: str, expected_job_step: str):
+    """Test reformatting job step name for each workflow."""
     # GIVEN a data analysis and a job step
 
     # WHEN reformation the job step
     reformatted_job_step: str = reformat_squeue_result_job_step(
-        data_analysis=data_analysis, job_step=raw_job_step
+        workflow=workflow, job_step=raw_job_step
     )
 
     # THEN it should return a reformatted job step
     assert reformatted_job_step == expected_job_step
 
 
 @pytest.mark.parametrize(
-    "data_analysis, raw_job_step, expected_job_step",
+    "workflow, raw_job_step, expected_job_step",
     [
-        (Pipeline.MIP_DNA, "jobstep", "jobstep"),
-        (Pipeline.MIP_RNA, "jobstep", "jobstep"),
-        (Pipeline.BALSAMIC, "job.step", "job.step"),
-        (Pipeline.SARS_COV_2, "jobstep", "jobstep"),
+        (Workflow.MIP_DNA, "jobstep", "jobstep"),
+        (Workflow.MIP_RNA, "jobstep", "jobstep"),
+        (Workflow.BALSAMIC, "job.step", "job.step"),
+        (Workflow.MUTANT, "jobstep", "jobstep"),
     ],
 )
 def test_reformat_squeue_result_job_step_malformed_job_step(
-    caplog, data_analysis: str, raw_job_step: str, expected_job_step: str
+    caplog, workflow: str, raw_job_step: str, expected_job_step: str
 ):
-    """Test reformatting job step name for each pipeline when job step is malformed."""
+    """Test reformatting job step name for each workflow when a job step is malformed."""
 
     caplog.set_level("ERROR")
     # GIVEN a data analysis and a job step
 
     # WHEN reformation the job step
     reformatted_job_step: str = reformat_squeue_result_job_step(
-        data_analysis=data_analysis, job_step=raw_job_step
+        workflow=workflow, job_step=raw_job_step
     )
 
     # THEN it should return the original job step

tests/fixtures/analysis-data.yaml

@@ -15,7 +15,7 @@ analyses:
     config_path: fixtures/runs/cuddlyhen/analysis/cuddlyhen_config.yaml
     type: wts
     user: paul.anderson@magnolia.com
-    data_analysis: RNAFUSION
+    workflow: RNAFUSION
     ticket_id: 123456
     workflow_manager: nf_tower
 
@@ -28,7 +28,7 @@ analyses:
     config_path: fixtures/runs/cuddlyhen/analysis/cuddlyhen_config.yaml
     type: wts
     user: paul.anderson@magnolia.com
-    data_analysis: RNAFUSION
+    workflow: RNAFUSION
     ticket_id: 123456
     workflow_manager: nf_tower
 
@@ -41,7 +41,7 @@ analyses:
     config_path: fixtures/runs/cuddlyhen/analysis/cuddlyhen_config.yaml
     type: wts
     user: paul.anderson@magnolia.com
-    data_analysis: RNAFUSION
+    workflow: RNAFUSION
     ticket_id: 123456
     workflow_manager: nf_tower
 
@@ -55,7 +55,7 @@ analyses:
     type: wgs
     user: paul.anderson@magnolia.com
     progress: .54
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA
     ticket_id: 123456
     workflow_manager: slurm
 
@@ -69,7 +69,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-RNA
+    workflow: MIP-RNA
 
   - case_id: lateraligator
     version: v4.2.0
@@ -80,7 +80,7 @@ analyses:
     config_path: fixtures/runs/politesnake/analysis/politesnake_config.yaml
     type: wes
     user: tom.cruise@magnolia.com
-    data_analysis: BALSAMIC
+    workflow: BALSAMIC
 
   - case_id: escapedgoat
     version: v3.6.0
@@ -92,7 +92,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA
 
   - case_id: crackpanda
     version: v3.6.0
@@ -104,7 +104,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: FLUFFY
+    workflow: FLUFFY
 
 
   - case_id: fancymole
@@ -117,7 +117,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA
 
 
   - case_id: trueferret
@@ -130,7 +130,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA
 
 
   - case_id: daringpidgeon
@@ -143,7 +143,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA
 
   - case_id: happycow
     version: v3.6.0
@@ -155,7 +155,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA
 
   - case_id: rarekitten
     version: v3.6.0
@@ -167,7 +167,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA
 
   - case_id: liberatedunicorn
     version: v3.6.0
@@ -179,7 +179,7 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA
 
   - case_id: emptydinosaur
     version: v3.6.0
@@ -191,4 +191,4 @@ analyses:
     config_path: fixtures/runs/nicemouse/analysis/nicemouse_config.yaml
     type: wgs
     user: paul.anderson@magnolia.com
-    data_analysis: MIP-DNA
+    workflow: MIP-DNA

tests/fixtures/tower/tower_workflow_completed.json

@@ -28,7 +28,7 @@
       "version": "2.2.0",
       "homePage": "https://github.com/nf-core/rnafusion",
       "gitmodules": null,
-      "description": "Nextflow rnafusion analysis pipeline, part of the nf-core community.",
+      "description": "Nextflow rnafusion analysis workflow, part of the nf-core community.",
       "name": "nf-core/rnafusion",
       "mainScript": "main.nf",
       "author": "Martin Proks, Annick Renevey"

tests/fixtures/tower/tower_workflow_running.json

@@ -28,7 +28,7 @@
       "version": "2.2.0",
       "homePage": "https://github.com/nf-core/rnafusion",
       "gitmodules": null,
-      "description": "Nextflow rnafusion analysis pipeline, part of the nf-core community.",
+      "description": "Nextflow rnafusion analysis workflow, part of the nf-core community.",
       "name": "nf-core/rnafusion",
       "mainScript": "main.nf",
       "author": "Martin Proks, Annick Renevey"

tests/integration/conftest.py

@@ -9,9 +9,9 @@
 from sqlalchemy.orm import Session
 
 from trailblazer.constants import (
-    PIPELINES,
     PRIORITY_OPTIONS,
     TYPES,
+    WORKFLOWS,
     JobType,
     TrailblazerStatus,
     WorkflowManager,
@@ -38,7 +38,7 @@ def client(flask_app: Flask) -> Generator[FlaskClient, None, None]:
 def analysis() -> Analysis:
     analysis = Analysis(
         config_path="config_path",
-        data_analysis="data_analysis",
+        workflow="workflow",
         case_id="case_id",
         out_dir="out_dir",
         priority=PRIORITY_OPTIONS[0],
@@ -80,15 +80,15 @@ def analysis() -> Analysis:
 
 @pytest.fixture
 def analyses() -> list[Analysis]:
-    """Analyses with different statuses, priorities, types, and pipelines."""
+    """Analyses with different statuses, priorities, types, and workflows."""
     analyses: list[Analysis] = []
     for priority in PRIORITY_OPTIONS:
         for type in TYPES:
             for status in TrailblazerStatus.statuses():
-                for pipeline in PIPELINES:
+                for workflow in WORKFLOWS:
                     analysis = Analysis(
                         config_path="config_path",
-                        data_analysis=pipeline,
+                        workflow=workflow,
                         case_id="case_id",
                         out_dir="out_dir",
                         priority=priority,
@@ -111,7 +111,7 @@ def analyses() -> list[Analysis]:
 def analysis_with_failed_job() -> Analysis:
     analysis = Analysis(
         config_path="config_path",
-        data_analysis="data_analysis",
+        workflow="workflow",
         case_id="case_id",
         out_dir="out_dir",
         priority=PRIORITY_OPTIONS[0],

tests/integration/endpoints/test_add_pending_analysis.py

@@ -14,7 +14,7 @@ def test_adding_analysis(client: FlaskClient, store: Store):
     create_analysis_request = CreateAnalysisRequest(
         case_id="case_id",
         config_path="config_path",
-        data_analysis=TrailblazerTypes.WGS,
+        workflow=TrailblazerTypes.WGS,
         priority=TrailblazerPriority.NORMAL,
         out_dir="out_dir",
         ticket="ticket_id",

tests/integration/endpoints/test_get_analyses.py

@@ -2,7 +2,7 @@
 
 from flask.testing import FlaskClient
 
-from trailblazer.constants import Pipeline, TrailblazerPriority, TrailblazerStatus
+from trailblazer.constants import TrailblazerPriority, TrailblazerStatus, Workflow
 from trailblazer.store.models import Analysis
 
 
@@ -108,21 +108,23 @@ def test_get_analyses_without_comments(client: FlaskClient, analyses: list[Analy
     assert len(response.json["analyses"]) == len(analyses_without_comments)
 
 
-def test_get_analyses_by_pipeline(client: FlaskClient, analyses: list[Analysis]):
-    # GIVEN analyses with different pipelines
+def test_get_analyses_by_workflow(client: FlaskClient, analyses: list[Analysis]):
+    # GIVEN analyses with different workflows
 
     # WHEN retrieving all mip analyses
-    response = client.get("/api/v1/analyses?pipeline=mip-dna")
+    response = client.get("/api/v1/analyses?workflow=mip-dna")
 
     # THEN it gives a success response
     assert response.status_code == HTTPStatus.OK
 
     # THEN it should only return mip analyses
-    assert all(a["data_analysis"] == Pipeline.MIP_DNA.lower() for a in response.json["analyses"])
+    assert all(
+        analysis["workflow"] == Workflow.MIP_DNA.lower() for analysis in response.json["analyses"]
+    )
 
 
 def test_get_analyses_by_order_id(client: FlaskClient, analyses: list[Analysis]):
-    # GIVEN analyses with different pipelines
+    # GIVEN analyses with different workflows
 
     # WHEN retrieving all analyses with order_id=0
     response = client.get("/api/v1/analyses?orderId=0")

tests/store/filters/test_analysis_filters.py

@@ -149,15 +149,15 @@ def test_filter_analyses_by_search_term_in_comment(
     assert existing_analysis == analyses.first()
 
 
-def test_filter_analyses_by_search_term_in_data_analysis(analysis_store: MockStore):
-    """Test return analysis when search term matches data analysis."""
+def test_filter_analyses_by_search_term_in_workflow(analysis_store: MockStore):
+    """Test return analysis when search term matches workflow."""
     # GIVEN a store containing analyses
     existing_analysis: Analysis = analysis_store.get_query(table=Analysis).first()
 
     # WHEN retrieving analyses by search term
     analyses: Query = filter_analyses_by_search_term(
         analyses=analysis_store.get_query(table=Analysis),
-        search_term=existing_analysis.data_analysis,
+        search_term=existing_analysis.workflow,
     )
 
     # THEN the analysis is a query

trailblazer/apps/slurm/api.py

@@ -86,9 +86,9 @@ def get_squeue_result(squeue_response: str) -> SqueueResult:
     return SqueueResult(jobs=squeue_response_content)
 
 
-def reformat_squeue_result_job_step(data_analysis: str, job_step: str) -> str:
+def reformat_squeue_result_job_step(workflow: str, job_step: str) -> str:
     """Standardise job step string according to data analysis."""
-    formatter: Callable = formatters.formatter_map.get(data_analysis, formatters.reformat_undefined)
+    formatter: Callable = formatters.formatter_map.get(workflow, formatters.reformat_undefined)
     return formatter(job_step=job_step)