Skip to content

Commit

Permalink
Removes FOLIO Migration report app, plugins, and misc bw migration code
Browse files Browse the repository at this point in the history
  • Loading branch information
jermnelson committed Apr 29, 2024
1 parent 8b47fb3 commit 0e37e14
Show file tree
Hide file tree
Showing 5 changed files with 0 additions and 332 deletions.
139 changes: 0 additions & 139 deletions libsys_airflow/plugins/folio/apps/folio_migration_view.py

This file was deleted.

66 changes: 0 additions & 66 deletions libsys_airflow/plugins/folio/helpers/bw.py
@@ -1,7 +1,5 @@
import datetime
import json
import logging
import pathlib

import requests

Expand Down Expand Up @@ -145,70 +143,6 @@ def email_failure(context):
send_email(to=to_addresses, subject=f"Error {ti.task_id}", html_content=html_body)


def discover_bw_parts_files(**kwargs):
airflow = kwargs.get("airflow", "/opt/airflow/")
jobs = int(kwargs["jobs"])
instance = kwargs["task_instance"]
iterations = pathlib.Path(airflow) / "migration/iterations"

bw_files = []
for iteration in iterations.iterdir():
bw_file = iteration / "results/boundwith_parts.json"
if bw_file.exists():
bw_files.append(str(bw_file))
else:
logger.error(f"{bw_file} doesn't exist")

shard_size = int(len(bw_files) / jobs)

for i in range(jobs):
start = i * shard_size
end = shard_size * (i + 1)
if i == jobs - 1:
end = len(bw_files)

instance.xcom_push(key=f"job-{i}", value=bw_files[start:end])

logger.info(f"Discovered {len(bw_files)} boundwidth part files for processing")


def check_add_bw(**kwargs):
instance = kwargs["task_instance"]
folio_client = kwargs["folio_client"]
job_number = kwargs["job"]

bw_file_parts = instance.xcom_pull(
task_ids="discovery-bw-parts", key=f"job-{job_number}"
)

logger.info(f"Started processing {len(bw_file_parts)}")

total_bw, total_errors = 0, 0
for file_path in bw_file_parts:
logger.info(f"Starting Boundwith processing for {file_path}")
bw, errors = 0, 0
with open(file_path) as fo:
for line in fo.readlines():
record = json.loads(line)
post_result = requests.post(
f"{folio_client.okapi_url}/inventory-storage/bound-with-parts",
headers=folio_client.okapi_headers,
json=record,
)
if post_result.status_code != 201:
errors += 1
logger.error(
f"Failed to post {record.get('id', 'NO ID')} from {file_path} error: {post_result.text}"
)
bw += 1

logger.info(f"Processed {file_path} added {bw} errors {errors}")
total_errors += errors
total_bw += bw

logger.info(f"Finished added {total_bw:,} boundwidths with {total_errors:,} errors")


def create_bw_record(**kwargs) -> dict:
"""
Creates a boundwidth record
Expand Down
9 changes: 0 additions & 9 deletions libsys_airflow/plugins/folio/main.py
Expand Up @@ -2,7 +2,6 @@
from flask import Blueprint
from libsys_airflow.plugins.folio.apps.circ_rules_tester_view import CircRulesTester
from libsys_airflow.plugins.folio.apps.healthcheck_view import Healthcheck
from libsys_airflow.plugins.folio.apps.folio_migration_view import FOLIOMigrationReports


bp = Blueprint(
Expand All @@ -13,13 +12,6 @@
static_url_path="/static/folio_plugin",
)

# FOLIO Migration Reports
folio_appbuilder_view = FOLIOMigrationReports()
folio_appbuilder_package = {
"name": "FOLIO Reports and Logs",
"category": "FOLIO",
"view": folio_appbuilder_view,
}

# Circ Rules Tester App
circ_rules_tester_view = CircRulesTester()
Expand All @@ -46,7 +38,6 @@ class FOLIOPlugin(AirflowPlugin):
executors = []
admin_views = []
appbuilder_views = [
folio_appbuilder_package,
circ_rules_tester_package,
healthcheck_package,
]
Expand Down
112 changes: 0 additions & 112 deletions tests/apps/test_folio_migration_view.py

This file was deleted.

6 changes: 0 additions & 6 deletions tests/helpers/test_boundwiths.py
@@ -1,5 +1,4 @@
import datetime
import json

import pytest
import requests
Expand All @@ -10,10 +9,8 @@

from libsys_airflow.plugins.folio.helpers.bw import (
add_admin_notes,
check_add_bw,
create_admin_note,
create_bw_record,
discover_bw_parts_files,
email_failure,
email_bw_summary,
post_bw_record,
Expand All @@ -25,8 +22,6 @@
MockTaskInstance,
)

import tests.mocks as mocks


@pytest.fixture
def mock_okapi_boundwith(monkeypatch, mocker: MockerFixture):
Expand Down Expand Up @@ -182,7 +177,6 @@ def test_create_bw_record_no_holdings(mock_folio_client):
assert bw_parts == {}



@pytest.fixture
def mock_context():
return {
Expand Down

0 comments on commit 0e37e14

Please sign in to comment.