Skip to content

Commit

Permalink
Merge pull request #467 from RockefellerArchiveCenter/issue-457
Browse files Browse the repository at this point in the history 
Improve unit tests for Accessions and API
  • Loading branch information
@helrond
helrond authored Jun 30, 2021
2 parents efd8d01 + e4e16e0 commit 1b26687
Show file tree
Hide file tree
Showing 12 changed files with 6,688 additions and 258 deletions.
16 changes: 6 additions & 10 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,27 +26,23 @@ RUN mkdir /var/run/clamav && \
chown clamav:clamav /var/run/clamav && \
chmod 750 /var/run/clamav && \
sed -i 's/^User .*$/User root/g' /etc/clamav/clamd.conf && \
sed -i 's/^DatabaseOwner .*$/DatabaseOwner root/g' /etc/clamav/freshclam.conf

# Update clamav databases
RUN wget -O /var/lib/clamav/main.cvd http://database.clamav.net/main.cvd && \
wget -O /var/lib/clamav/daily.cvd http://database.clamav.net/daily.cvd && \
wget -O /var/lib/clamav/bytecode.cvd http://database.clamav.net/bytecode.cvd && \
chown clamav:clamav /var/lib/clamav/*.cvd

sed -i 's/^DatabaseOwner .*$/DatabaseOwner root/g' /etc/clamav/freshclam.conf && \
freshclam

# Set up SSH
RUN mkdir /run/sshd && cp -r /etc/ssh /etc/ssh2

# Copy Aurora application files
# Install Python dependencies
RUN mkdir -p /code/
COPY . /code
COPY requirements.txt /code

RUN mkdir -p /data/

# Install Python modules
RUN pip install --upgrade pip && pip install -r /code/requirements.txt

COPY . /code

EXPOSE 8000
EXPOSE 22

Expand Down
2 changes: 1 addition & 1 deletion aurora/bag_transfer/accession/forms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class Meta:
fields=("name", "type"),
extra=0,
widgets={
"name": forms.widgets.TextInput(attrs={"class": "form-control col-sm-8", "disabled": "True"}),
"name": forms.widgets.TextInput(attrs={"class": "form-control col-sm-8"}),
"type": forms.widgets.Select(attrs={"class": "form-control"}),
},
)
143 changes: 74 additions & 69 deletions aurora/bag_transfer/accession/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,15 +104,8 @@ def post(self, request, *args, **kwargs):
form.process_status = Accession.CREATED
accession = form.save()
creators_formset.save()
merged_rights_statements = RightsStatement.merge_rights(rights_statements)
for statement in merged_rights_statements:
statement.accession = accession
statement.save()
for transfer in transfers_list:
BAGLog.log_it("BACC", transfer)
transfer.process_status = Archives.ACCESSIONING_STARTED
transfer.accession = accession
transfer.save()
self.update_accession_rights(RightsStatement.merge_rights(rights_statements), accession)
self.update_accession_transfers(transfers_list, accession)
messages.success(request, "
Accession created successfully!")
if settings.DELIVERY_URL:
try:
Expand All @@ -134,7 +127,6 @@ def post(self, request, *args, **kwargs):
accession.save()
messages.success(request, "Accession data delivered.")
except Exception as e:
print(e)
messages.error(
request, "Error delivering accession data: {}".format(e)
)
Expand Down Expand Up @@ -165,64 +157,17 @@ def get(self, request, *args, **kwargs):
rights_statements = (
RightsStatement.objects.filter(archive__in=id_list)
.annotate(rights_group=F("rights_basis"))
.order_by("rights_group")
)
# should this get the source_organization from bag_data instead? Need to coordinate with data in other views
.order_by("rights_group"))
organization = transfers_list[0].organization
notes = {"appraisal": []}
dates = {"start": [], "end": []}
creators_list = []
descriptions_list = []
languages_list = []
extent_files = 0
extent_size = 0
for transfer in transfers_list:
bag_data = transfer.get_bag_data()
extent_size = extent_size + int(
bag_data.get("payload_oxum", "0.0").split(".")[0]
)
extent_files = extent_files + int(
bag_data.get("payload_oxum", "0.0").split(".")[1]
)
dates["start"].append(bag_data.get("date_start", ""))
dates["end"].append(bag_data.get("date_end", ""))
notes["appraisal"].append(bag_data.get("appraisal_note", ""))
descriptions_list.append(
bag_data.get("internal_sender_description", "")
)
for language in bag_data.get("language", []):
languages_list.append(language)
creators_list += transfer.get_records_creators()
for statement in rights_statements:
rights_info = statement.get_rights_info_object()
rights_granted = statement.get_rights_granted_objects()
if not statement.rights_basis.lower() in notes:
notes[statement.rights_basis.lower()] = []
notes[statement.rights_basis.lower()].append(
next(
value
for key, value in rights_info.__dict__.items()
if "_note" in key.lower()
)
)
for grant in rights_granted:
notes[statement.rights_basis.lower()].append(
grant.rights_granted_note
)
record_creators = list(set(creators_list))
languages_list = list(set(languages_list))
language = LanguageCode.objects.get_or_create(code="und")[0]
if len(languages_list) == 1:
LanguageCode.objects.get_or_create(code=languages_list[0])[0]
if len(languages_list) > 1:
language = LanguageCode.objects.get_or_create(code="mul")[0]
title = "{} {}".format(organization, bag_data.get("record_type", ""))
if len(record_creators) > 0:
title = "{}, {} {}".format(
organization,
", ".join([creator.name for creator in record_creators]),
bag_data.get("record_type", ""),
)
creators_list = list(set([c for t in transfers_list for c in t.get_records_creators()]))
notes, dates, descriptions_list, languages_list, extent_files, extent_size, record_type = self.grouped_transfer_data(transfers_list)
notes.update(self.rights_statement_notes(rights_statements))
language = self.parse_language(languages_list)
title = self.parse_title(
organization,
record_type,
", ".join([creator.name for creator in creators_list]))

form = AccessionForm(
initial={
"title": title,
Expand All @@ -239,11 +184,11 @@ def get(self, request, *args, **kwargs):
"appraisal_note": " ".join(set(notes.get("appraisal", []))),
"organization": organization,
"language": language,
"creators": record_creators,
"creators": creators_list,
}
)
creators_formset = CreatorsFormSet(
queryset=RecordCreators.objects.filter(name__in=record_creators)
queryset=RecordCreators.objects.filter(name__in=creators_list)
)
return render(
request,
Expand All @@ -257,6 +202,26 @@ def get(self, request, *args, **kwargs):
},
)

def grouped_transfer_data(self, transfers_list):
notes = {"appraisal": []}
dates = {"start": [], "end": []}
descriptions_list = []
languages_list = []
extent_files = 0
extent_size = 0
for transfer in transfers_list:
bag_data = transfer.get_bag_data()
extent_size = extent_size + int(bag_data.get("payload_oxum", "0.0").split(".")[0])
extent_files = extent_files + int(bag_data.get("payload_oxum", "0.0").split(".")[1])
dates["start"].append(bag_data.get("date_start", ""))
dates["end"].append(bag_data.get("date_end", ""))
notes["appraisal"].append(bag_data.get("appraisal_note", ""))
descriptions_list.append(bag_data.get("internal_sender_description", ""))
languages_list += [l for l in bag_data.get("language", [])]
languages_list = list(set(languages_list))
record_type = bag_data.get("record_type", "")
return notes, dates, descriptions_list, languages_list, extent_files, extent_size, record_type

def handle_ajax_request(self, request):
"""Handles JavaScript AJAX requests."""
rdata = {}
Expand All @@ -277,6 +242,46 @@ def handle_ajax_request(self, request):
rdata["error"] = str(e)
return self.render_to_json_response(rdata)

def parse_language(self, languages_list):
if len(languages_list) == 1:
language = LanguageCode.objects.get_or_create(code=languages_list[0])[0]
elif len(languages_list) > 1:
language = LanguageCode.objects.get_or_create(code="mul")[0]
else:
language = LanguageCode.objects.get_or_create(code="und")[0]
return language

def parse_title(self, organization, record_type, creators_list):
return ("{}, {} {}".format(organization, creators_list, record_type)
if len(creators_list) > 0 else "{} {}".format(organization, record_type))

def rights_statement_notes(self, rights_statements):
notes = {}
for statement in rights_statements:
rights_info = statement.get_rights_info_object()
rights_granted = statement.get_rights_granted_objects()
if not statement.rights_basis.lower() in notes:
notes[statement.rights_basis.lower()] = []
notes[statement.rights_basis.lower()].append(
next(value for key, value in rights_info.__dict__.items()
if "_note" in key.lower()))
for grant in rights_granted:
notes[statement.rights_basis.lower()].append(
grant.rights_granted_note)
return notes

def update_accession_rights(self, merged_rights_statements, accession):
for statement in merged_rights_statements:
statement.accession = accession
statement.save()

def update_accession_transfers(self, transfers_list, accession):
for transfer in transfers_list:
BAGLog.log_it("BACC", transfer)
transfer.process_status = Archives.ACCESSIONING_STARTED
transfer.accession = accession
transfer.save()


class SavedAccessionsDatatableView(ArchivistMixin, BaseDatatableView):
"""Handles processing of requests for Accessions in datatable, making page
Expand Down
4 changes: 1 addition & 3 deletions aurora/bag_transfer/api/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,6 @@ class BagItProfileViewSet(viewsets.ReadOnlyModelViewSet):
def get_serializer_class(self):
if self.action == "list":
return BagItProfileListSerializer
if self.action == "retrieve":
return BagItProfileSerializer
return BagItProfileSerializer


Expand Down Expand Up @@ -139,7 +137,7 @@ class AccessionViewSet(
"""Endpoint for Accessions"""

def get_queryset(self):
queryset = Accession.objects.all()
queryset = Accession.objects.all().order_by("-created")
if not self.request.user.is_archivist():
queryset = queryset.filter(organization=self.request.user.organization)
process_status = self.request.GET.get("process_status", "")
Expand Down
Loading

0 comments on commit 1b26687

Please sign in to comment.