Skip to content

Commit

Permalink
unit tests for cron jobs, fixes #376
Browse files Browse the repository at this point in the history
  • Loading branch information
@helrond
helrond committed Jan 3, 2020
1 parent 0786d70 commit e3fbb1d
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 69 deletions.
140 changes: 74 additions & 66 deletions aurora/bag_transfer/lib/cron.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ class DiscoverTransfers(CronJobBase):
code = "transfers.discover_transfers"

def do(self):
result = True
Pter.cron_open(self.code)
BAGLog.log_it("CSTR")

Expand All @@ -29,78 +30,82 @@ def do(self):
if to_process:

for upload_list in to_process:

email = Mailer()

machine_file_identifier = Archives().gen_identifier()
org = Organization.is_org_active(upload_list["org"])
user = User.is_user_active(upload_list["upload_user"], org)

email.to = [user.email]

new_arc = Archives.initial_save(
org,
user,
upload_list["file_path"],
upload_list["file_size"],
upload_list["file_modtime"],
machine_file_identifier,
upload_list["file_type"],
upload_list["bag_it_name"],
)

BAGLog.log_it("ASAVE", new_arc)
print(
"\nValidating transfer {}".format(new_arc.machine_file_identifier)
)

if upload_list["auto_fail"]:
new_arc.setup_save(upload_list)
new_arc.process_status = Archives.INVALID
BAGLog.log_it(upload_list["auto_fail_code"], new_arc)
email.setup_message("TRANS_FAIL_VAL", new_arc)
email.send()
FH.remove_file_or_dir(new_arc.machine_file_path)

else:
bag = bagChecker(new_arc)
if bag.bag_passed_all():
print(
"Transfer {} is valid".format(
new_arc.machine_file_identifier
)
)
new_arc.process_status = Archives.VALIDATED
new_arc.bag_it_valid = True
BAGLog.log_it("APASS", new_arc)
email.setup_message("TRANS_PASS_ALL", new_arc)
email.send()
FH.move_file_or_dir(
"/data/tmp/{}".format(new_arc.bag_it_name),
"{}{}".format(
settings.STORAGE_ROOT_DIR,
new_arc.machine_file_identifier,
),
)
FH.remove_file_or_dir(new_arc.machine_file_path)
new_arc.machine_file_path = "{}{}".format(
settings.STORAGE_ROOT_DIR, new_arc.machine_file_identifier
)
else:
print(
"Transfer {} is invalid".format(
new_arc.machine_file_identifier
)
)
try:
email = Mailer()

machine_file_identifier = Archives().gen_identifier()
org = Organization.is_org_active(upload_list["org"])
user = User.is_user_active(upload_list["upload_user"], org)

email.to = [user.email]

new_arc = Archives.initial_save(
org,
user,
upload_list["file_path"],
upload_list["file_size"],
upload_list["file_modtime"],
machine_file_identifier,
upload_list["file_type"],
upload_list["bag_it_name"],
)

BAGLog.log_it("ASAVE", new_arc)
print(
"\nValidating transfer {}".format(new_arc.machine_file_identifier)
)

if upload_list["auto_fail"]:
new_arc.setup_save(upload_list)
new_arc.process_status = Archives.INVALID
BAGLog.log_it(bag.ecode, new_arc)
BAGLog.log_it(upload_list["auto_fail_code"], new_arc)
email.setup_message("TRANS_FAIL_VAL", new_arc)
email.send()
FH.remove_file_or_dir(new_arc.machine_file_path)

new_arc.save()
FH.remove_file_or_dir("/data/tmp/{}".format(new_arc.bag_it_name))
else:
bag = bagChecker(new_arc)
if bag.bag_passed_all():
print(
"Transfer {} is valid".format(
new_arc.machine_file_identifier
)
)
new_arc.process_status = Archives.VALIDATED
new_arc.bag_it_valid = True
BAGLog.log_it("APASS", new_arc)
email.setup_message("TRANS_PASS_ALL", new_arc)
email.send()
FH.move_file_or_dir(
"/data/tmp/{}".format(new_arc.bag_it_name),
"{}{}".format(
settings.STORAGE_ROOT_DIR,
new_arc.machine_file_identifier,
),
)
FH.remove_file_or_dir(new_arc.machine_file_path)
new_arc.machine_file_path = "{}{}".format(
settings.STORAGE_ROOT_DIR, new_arc.machine_file_identifier
)
else:
print(
"Transfer {} is invalid".format(
new_arc.machine_file_identifier
)
)
new_arc.process_status = Archives.INVALID
BAGLog.log_it(bag.ecode, new_arc)
email.setup_message("TRANS_FAIL_VAL", new_arc)
email.send()
FH.remove_file_or_dir(new_arc.machine_file_path)

new_arc.save()
FH.remove_file_or_dir("/data/tmp/{}".format(new_arc.bag_it_name))
except Exception as e:
print("Error discovering transfer {}: {}".format(machine_file_identifier, str(e)))
result = False
Pter.cron_close(self.code)
return result


class DeliverTransfers(CronJobBase):
Expand All @@ -110,6 +115,7 @@ class DeliverTransfers(CronJobBase):
code = "transfers.deliver_transfers"

def do(self):
result = True
Pter.cron_open(self.code)
for archive in Archives.objects.filter(
process_status=Archives.ACCESSIONING_STARTED
Expand Down Expand Up @@ -163,5 +169,7 @@ def do(self):
archive.save()
except Exception as e:
print("Error delivering transfer {}: {}".format(archive, str(e)))
result = False

Pter.cron_close(self.code)
return result
8 changes: 5 additions & 3 deletions aurora/bag_transfer/test/helpers.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from datetime import datetime
from os import path, listdir, rename
from os import path, listdir, rename, chown
import pwd
import random
import string
from django.contrib.auth.models import Group
Expand Down Expand Up @@ -204,7 +205,7 @@ def create_test_archive(transfer, org):

# Creates target bags to be picked up by a TransferRoutine based on a string.
# This allows processing of bags serialized in multiple formats at once.
def create_target_bags(target_str, test_bags_dir, org):
def create_target_bags(target_str, test_bags_dir, org, username):
moved_bags = []
target_bags = [b for b in listdir(test_bags_dir) if b.startswith(target_str)]
if len(target_bags) < 1:
Expand All @@ -223,7 +224,8 @@ def create_target_bags(target_str, test_bags_dir, org):
index += 1

# chowning path to root
FH.chown_path_to_root(new_path)
# FH.chown_path_to_root(new_path)
chown(new_path, pwd.getpwnam(username).pw_uid, -1)
moved_bags.append(new_path)

return moved_bags
Expand Down
39 changes: 39 additions & 0 deletions aurora/bag_transfer/test/test_cron.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import os
import pwd
import random

from django.test import TransactionTestCase, Client
from django.conf import settings

from bag_transfer.lib.cron import DiscoverTransfers, DeliverTransfers
from bag_transfer.test import helpers
from bag_transfer.test.setup import bags_ref, TEST_ORG_COUNT
from bag_transfer.models import Archives, User


class CronTestCase(TransactionTestCase):
def setUp(self):
self.orgs = helpers.create_test_orgs(org_count=1)
self.baglogcodes = helpers.create_test_baglogcodes()
self.groups = helpers.create_test_groups(['managing_archivists'])
self.user = helpers.create_test_user(username=settings.TEST_USER['USERNAME'], org=random.choice(self.orgs))
for group in self.groups:
self.user.groups.add(group)
self.user.is_staff = True
self.user.set_password(settings.TEST_USER['PASSWORD'])
self.user.save()
self.client = Client()

def test_cron(self):
for ref in bags_ref:
helpers.create_target_bags(ref[0], settings.TEST_BAGS_DIR, self.orgs[0], username=self.user.username)
discovered = DiscoverTransfers().do()
self.assertIsNot(False, discovered)
for archive in Archives.objects.filter(process_status=Archives.VALIDATED):
archive.process_status = Archives.ACCESSIONING_STARTED
archive.save()
delivered = DeliverTransfers().do()
self.assertIsNot(False, delivered)

def tearDown(self):
helpers.delete_test_orgs(self.orgs)

0 comments on commit e3fbb1d

Please sign in to comment.