WIP: Demonstrate MVP integration of EaaSI UVI

docker-compose.yml

@@ -16,6 +16,7 @@ services:
       - ES_HOSTS=elasticsearch:9200
       - CELERY_BROKER_URL=redis://redis:6379
       - SS_HOSTS=http://test:test@192.168.1.128:62081
+      - EAASI_HOST=https://eaas.archivematica.net
     volumes:
       - node_modules:/src/node_modules
       - .:/src

scope/eaas_uvi.py

@@ -0,0 +1,124 @@
+import logging
+import json
+import sys
+import time
+
+import requests
+from requests.adapters import HTTPAdapter, Retry
+
+
+MODULE_NAME = "eaas_uvi" if __name__ == "__main__" else __name__
+logger = logging.getLogger(MODULE_NAME)
+logger.setLevel(logging.INFO)
+handler = logging.StreamHandler(stream=sys.stdout)
+formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+
+
+class ResultNotFound(Exception):
+    """Custom exception for when no result is returned."""
+
+
+class EaaSUVIClient:
+
+    DATA_TYPES = ("zip", "tar", "bagit+zip", "bagit+tar")
+
+    def __init__(self, base_url, data_url, data_type="zip"):
+        self.base_url = base_url
+        self.api_base_url = "{}/emil/environment-proposer/api/v2".format(
+            base_url.rstrip("/")
+        )
+        self.data_url = data_url
+
+        self.data_type = "zip"
+        if data_type in self.DATA_TYPES:
+            self.data_type = data_type
+
+        self.results = {}
+
+    def _submit_proposal(self):
+        """Submit proposal to environment-proposer endpoint and return id."""
+        url = "{}/proposals".format(self.api_base_url)
+        response = requests.post(
+            url, json={"data_url": self.data_url, "data_type": self.data_type},
+        )
+        if not response.status_code == 202:
+            err_msg = "Error submitting environment proposer request to URL {}. Status code: {}".format(
+                url, response.status_code
+            )
+            raise ResultNotFound(err_msg)
+
+        response_dict = json.loads(response.content)
+        return response_dict.get("id")
+
+    def _poll_until_result_ready(self, proposal_id):
+        """Poll waitqueue endpoint until results are ready and return location_url."""
+        status_url = "{}/waitqueue/{}".format(self.api_base_url, proposal_id)
+
+        # Configure requests.Session to retry on connectivity issues and 200
+        # status code, which indicates results are not yet ready.
+        session = requests.Session()
+        retries = Retry(total=15, backoff_factor=1, status_forcelist=[200])
+        session.mount("https://", HTTPAdapter(max_retries=retries))
+        session.mount("http://", HTTPAdapter(max_retries=retries))
+
+        response = session.get(status_url, allow_redirects=False)
+
+        if response.status_code != 303:
+            err_msg = "Error reported by {}. Status code: {}. Body: {}".format(
+                status_url, response.status_code, response.content
+            )
+            raise ResultNotFound(err_msg)
+
+        return response.headers.get("Location")
+
+    def _fetch_result(self, location_url):
+        """Fetch result from URL returned in location header and save as dict."""
+        response = requests.get(location_url)
+        if not response.status_code == 200:
+            err_msg = "Error fetching result from UVI. Status code: {}. Body: {}".format(
+                response.status_code, response.content
+            )
+            raise ResultNotFound(err_msg)
+        self.results = json.loads(response.content)
+        return self.results
+
+    def get_recommendations(self):
+        """Submit files to UVI API and return results when ready."""
+        proposal_id = self._submit_proposal()
+        logger.info("Proposal {} submitted.".format(proposal_id))
+
+        time.sleep(3)
+
+        location_url = self._poll_until_result_ready(proposal_id)
+
+        logger.info("Fetching result from {}".format(location_url))
+
+        self._fetch_result(location_url)
+        logger.info("Result: {}".format(self.results))
+
+        return self.results
+
+    def parse_suggested_environments_from_results(self):
+        """Parse results returned by UVI API."""
+        if not self.results:
+            raise ResultNotFound("Recommendations not yet fetched from UVI.")
+
+        # TODO: Do something with results!
+        # "suggested" : [ {
+        #   "id" : "...",
+        #   "label" : "...",
+        #   "defaultEnvironment" : { }
+        # }, {
+        #   "id" : "...",
+        #   "label" : "...",
+        #   "defaultEnvironment" : { }
+        # } ],
+        suggested_environments = self.results.get("result").get("suggested")
+        if not suggested_environments:
+            logger.warning("No suggested environments found.")
+            return
+        for suggestion in suggested_environments:
+            logger.info("Suggestion: {}".format(suggestion))
+        return suggested_environments

scope/settings.py

@@ -211,3 +211,7 @@
 SS_HOSTS = env(
     "SS_HOSTS", cast=list, subcast=str, postprocessor=ss_hosts_parser, default=[]
 )
+
+# EaaSI
+
+EAASI_HOST = env("EAASI_HOST")

scope/urls.py

@@ -47,6 +47,7 @@
     url(r"^folder/(?P<pk>\d+)/delete/$", views.delete_dip, name="delete_dip"),
     url(r"^folder/(?P<pk>\d+)/$", views.dip, name="dip"),
     url(r"^folder/(?P<pk>\d+)/download$", views.download_dip, name="download_dip"),
+    url(r"^folder/(?P<pk>\d+)/recommend$", views.recommend_dip_environments, name="recommend_dip_environments"),
     url(r"^object/(?P<pk>[-\w-]+)$", views.digital_file, name="digital_file"),
     url(r"^new_folder/", views.new_dip, name="new_dip"),
     url(r"^orphan_folders/", views.orphan_dips, name="orphan_dips"),

scope/views.py

@@ -20,6 +20,7 @@
 from search.helpers import add_digital_file_filters
 from search.helpers import add_query_to_search
 
+from .eaas_uvi import EaaSUVIClient, ResultNotFound
 from .forms import ContentForm
 from .forms import DeleteByDublinCoreForm
 from .forms import DublinCoreSettingsForm
@@ -691,6 +692,36 @@ def download_dip(request, pk):
     return response
 
 
+@login_required(login_url="/login/")
+def recommend_dip_environments(request, pk):
+    dip = get_object_or_404(DIP, pk=pk)
+    if not django_settings.EAASI_HOST:
+        raise RuntimeError("Configuration not found for EaaSI host.")
+    if dip.objectszip:
+        # TODO: Upload DIP so it's fetchable via URL by EaaSI server.
+        # For now, we fake it with DATA_URL.
+        # With a real DIP, how do we make sure that EaaSI is basing its
+        # recommendations on the actual digital objects, and not e.g. the METS?
+        DATA_URL = "https://github.com/tw4l/sample-data/blob/main/fmt-38.zip?raw=true"
+        eaas_client = EaaSUVIClient(
+            base_url=django_settings.EAASI_HOST,
+            data_url=DATA_URL
+        )
+        try:
+            results = eaas_client.get_recommendations()
+        except (requests.ConnectionError, ResultNotFound) as err:
+            results = "Error fetching recommendations: {}".format(err)
+        return render(
+            request,
+            "dip_recommendations.html",
+            {
+                "dip": dip,
+                "api_results": results,
+            },
+        )
+    return redirect("dip", pk=pk)
+
+
 @login_required(login_url="/login/")
 def settings(request):
     if not request.user.is_superuser:

templates/dip.html

@@ -43,6 +43,7 @@ <h2 class="mb-3">{% trans "Attachments" %}</h2>
           </p>
           <p>{% trans "By clicking on the button below you'll download all the digital files included in this folder." %}</p>
           <a href="{% url 'download_dip' dip.pk %}" class="btn btn-primary">{% trans "Download DIP" %}</a>
+          <a href="{% url 'recommend_dip_environments' dip.pk %}" class="btn btn-primary">{% trans "Recommend emulation environments" %}</a>
         </div>
       </div>
     </div>

templates/dip_recommendations.html

@@ -0,0 +1,26 @@
+{% extends 'base.html' %}
+
+{% load custom_tags %}
+{% load i18n %}
+
+{% block title %}
+{% blocktrans %}Folder {{ dip }} - EaaSI UVI recommendations{% endblocktrans %}
+{% endblock %}
+
+{% block content %}
+  <ol class="breadcrumb">
+    {% trans "Untitled" as untitled %}
+    <li class="breadcrumb-item"><a href="{% url 'collections' %}">{% trans "Collections" %}</a></li>
+    <li class="breadcrumb-item"><a href="{% url 'collection' dip.collection.pk %}">{{ dip.collection.dc.title|default:untitled }}</a></li>
+    <li class="breadcrumb-item"><a href="{% url 'dip' dip.pk %}">{{ dip.dc.title|default:untitled }}</a></li>
+    <li class="breadcrumb-item active">{% trans "Emulation environment recommendations" %}</li>
+  </ol>
+  <div class="card mt-4">
+    <div class="card-header">
+      <h5 class="mb-0">{% trans "EaaSI UVI recommendations" %}</h5>
+    </div>
+    <div class="card-body">
+      <p>{{ api_results }}</p>
+    </div>
+  </div>
+{% endblock %}