Add code coverage with github actions

.github/workflows/python-app.yml

@@ -11,6 +11,8 @@ on:
 
 env:
   PYTHONPATH: /Users/Shared/folio_migration_tools
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
 
 jobs:
   build:
@@ -33,4 +35,11 @@ jobs:
         git clone https://github.com/FOLIO-FSE/folio_migration_tools.git /Users/Shared/folio_migration_tools --depth=2
     - name: Test with pytest
       run: |
-        pytest
+        coverage run -m pytest
+        coverage lcov
+        coveralls --service=github
+    # - name: Send code coverage
+    #   uses: coverallsapp/github-action@master
+    #   with:
+    #     github-token: ${{ secrets.GITHUB_TOKEN }}
+    #     path-to-lcov: ${{ github.workspace }}/.coverage

.gitignore

@@ -1,3 +1,6 @@
 log*
 .env
 __pycache__/
+.coveralls.yml
+.coverage
+coverage.lcov

Dockerfile

@@ -3,6 +3,7 @@ FROM apache/airflow:2.2.4-python3.9
 USER root
 RUN usermod -u 214 airflow
 
+
 RUN apt-get -y update && apt-get -y install git
 
 ENV PYTHONPATH "${PYTHONPATH}:/opt/airflow/folio_migration_tools"

README.md

@@ -1,3 +1,5 @@
+[![Coverage Status](https://coveralls.io/repos/github/sul-dlss/libsys-airflow/badge.svg?branch=main)](https://coveralls.io/github/sul-dlss/libsys-airflow?branch=main)
+
 # libsys-airflow
 Airflow DAGS for libsys processes and migrating ILS data into FOLIO
 
@@ -54,5 +56,8 @@ the location of where you have local clone repository of the
 
 `PYTHONPATH='{path-to-folio_migration_tools}' pytest`
 
+To run with code coverage:
+`PYTHONPATH=../folio_migration_tools/ coverage run -m pytest`
+
 ## Symphony Mount
 MARC data to be converted will be mounted on the sul-libsys-airflow server under `/sirsi_prod` which is a mount of `/s/SUL/Dataload/Folio` on the Symphony server.

dags/bib_records.py

@@ -40,6 +40,11 @@
     post_folio_items_records
 )
 
+from plugins.folio.marc import (
+    post_marc_to_srs,
+    replace_srs_record_type
+)
+
 logger = logging.getLogger(__name__)
 
 sul_config = LibraryConfiguration(
@@ -242,6 +247,11 @@ def marc_only(*args, **kwargs):
             },
         )
 
+        update_record_type_srs = PythonOperator(
+            task_id="update-record-type-srs",
+            python_callable=replace_srs_record_type,
+        )
+
         finish_conversion = DummyOperator(
             task_id="finished-conversion",
             trigger_rule="none_failed_or_skipped",
@@ -253,6 +263,7 @@ def marc_only(*args, **kwargs):
             >> convert_instances_valid_json
             >> finish_conversion
         )
+        convert_marc_to_folio_instances >> update_record_type_srs >> finish_conversion  # noqa
         marc_only_convert_check >> [convert_tsv_to_folio_holdings, finish_conversion]  # noqa
         (
             convert_tsv_to_folio_holdings
@@ -285,6 +296,16 @@ def marc_only(*args, **kwargs):
 
             login >> post_instances >> finish_instances
 
+        marc_to_srs = PythonOperator(
+            task_id="marc-to-srs",
+            python_callable=post_marc_to_srs,
+            op_kwargs={
+                "library_config": sul_config,
+            }
+        )
+
+        finish_instances >> marc_to_srs >> finished_all_posts
+
         marc_only_post_check = BranchPythonOperator(
             task_id="marc-only-post-check",
             python_callable=marc_only,

plugins/folio/marc.py

@@ -0,0 +1,57 @@
+import json
+import logging
+import pathlib
+
+from migration_tools.migration_tasks.batch_poster import BatchPoster
+
+
+def post_marc_to_srs(*args, **kwargs):
+    dag = kwargs["dag_run"]
+
+    task_config = BatchPoster.TaskConfiguration(
+        name="marc-to-srs-batch-poster",
+        migration_task_type="BatchPoster",
+        object_type="SRS",
+        file={
+            "file_name": f"folio_srs_instances_{dag.run_id}_bibs-transformer.json"  # noqa
+        },
+        batch_size=kwargs.get("MAX_ENTITIES", 1000),
+    )
+
+    library_config = kwargs["library_config"]
+    library_config.iteration_identifier = dag.run_id
+
+    srs_batch_poster = BatchPoster(
+        task_config,
+        library_config,
+        use_logging=False
+    )
+
+    srs_batch_poster.do_work()
+
+    srs_batch_poster.wrap_up()
+
+    logging.info("Finished posting MARC json to SRS")
+
+
+def replace_srs_record_type(*args, **kwargs):
+    dag = kwargs["dag_run"]
+
+    airflow = kwargs.get("airflow", "/opt/airflow")
+    airflow_path = pathlib.Path(airflow)
+
+    srs_file = (
+        airflow_path
+        / f"migration/results/folio_srs_instances_{dag.run_id}_bibs-transformer.json"  # noqa
+    )
+
+    marc_recs = []
+    with open(srs_file) as fo:
+        for row in fo.readlines():
+            srs_marc = json.loads(row)
+            srs_marc["recordType"] = "MARC"
+            marc_recs.append(srs_marc)
+
+    with open(srs_file, "w+") as fo:
+        for row in marc_recs:
+            fo.write(f"{json.dumps(row)}\n")

plugins/tests/test_marc.py

@@ -0,0 +1,23 @@
+import json
+
+from plugins.folio.marc import replace_srs_record_type
+from plugins.tests.test_helpers import mock_dag_run, mock_file_system  # noqa
+
+
+def test_replace_srs_record_type(mock_dag_run, mock_file_system):  # noqa
+    dag = mock_dag_run
+    airflow_path = mock_file_system[0]
+    results_path = mock_file_system[3]
+
+    # Mock SRS Record
+    srs_mock_filename = (
+        f"folio_srs_instances_{dag.run_id}_bibs-transformer.json"  # noqa
+    )
+    srs_mock_file = results_path / srs_mock_filename
+    srs_mock_file.write_text("""{ "recordType": "MARC_BIB" }""")
+
+    replace_srs_record_type(airflow=airflow_path, dag_run=dag)
+
+    srs_json_doc = json.loads(srs_mock_file.read_text())
+
+    assert srs_json_doc["recordType"] == "MARC"

requirements.txt

@@ -1,5 +1,7 @@
 apache-airflow
 argparse_prompt
+coverage
+coveralls
 flake8
 folioclient
 folio-uuid