Merge branch 'ui' into 'master'

README.md

@@ -3,14 +3,9 @@
 ReproServer
 ===========
 
-Goals
------
+An online service to run ReproZip bundles in the cloud. No need to install reprounzip on your machine, point ReproServer to a RPZ file or upload one, and interact with the reproduced environment from your browser.
 
-* Import something we can build a Docker image from (currently only a ReproZip package)
-* Build a Docker image from it
-* Allow the user to change experiment parameters and input files
-* Run the experiment
-* Show the log and output files to the user
+Additionally, ReproServer can capture remote web assets referenced by a web application that has been packaged as a RPZ file. That way, you will always be able to get a consistent reproduction of the RPZ bundle, even if those remote assets disappear.
 
 How to run this with Tilt
 -------------------------
@@ -25,13 +20,13 @@ For example, create a local cluster with:
 minikube start --kubernetes-version=1.22.2 --driver=docker --nodes=1 --container-runtime=docker --ports=8000:30808
 ```
 
-Install the ingress controller using::
+Install the ingress controller using:
 
 ```
 kubectl apply -f k8s/nginx-ingress.k8s-1.22.yml
 ```
 
-Start the application for development using::
+Start the application for development using:
 
 ```
 tilt up

reproserver/web/__init__.py

@@ -31,8 +31,8 @@ def make_app(debug=False, xsrf_cookies=True, proxy=None):
             URLSpec('/results/([^/]+)', views.Results, name='results'),
             URLSpec('/results/([^/]+)/json', views.ResultsJson,
                     name='results_json'),
-            URLSpec('/web/([^/]+)', webcapture.Dashboard,
-                    name='webcapture_dashboard'),
+            URLSpec('/web/([^/]+)', webcapture.Index,
+                    name='webcapture_index'),
             URLSpec('/web/([^/]+)/preview', webcapture.Preview,
                     name='webcapture_preview'),
             URLSpec('/web/([^/]+)/record', webcapture.StartRecord,
@@ -50,6 +50,8 @@ def make_app(debug=False, xsrf_cookies=True, proxy=None):
                     name='webcapture_upload_wacz'),
             URLSpec('/web/([^/]+)/download', webcapture.Download,
                     name='webcapture_download'),
+            URLSpec('/web/([^/]+)/done/([^/]+)', webcapture.Done,
+                    name='webcapture_done'),
             URLSpec('/about', views.About, name='about'),
             URLSpec('/data', views.Data, name='data'),
             URLSpec('/health', views.Health, name='health'),

reproserver/web/templates/setup.html

@@ -9,7 +9,7 @@
   </ol>
 </nav>
 
-<h2>Package {{ filename | truncate(60) }}</h2>
+<h1>Package {{ filename | truncate(60) }}</h1>
 
 {% if repo_url -%}
 <p>From <a href="{{ repo_url }}">{{ repo_name }}</a></p>
@@ -26,7 +26,7 @@ <h3 class="card-header">
     {% else %}
     <p>If this RPZ is a web application, you can use ReproServer to capture the remote assets that are referenced by it (CSS, JS, etc). This will allow the application to keep working correctly in web browsers if those remote sources go away.</p>
     {% endif %}
-    <a href="{{ reverse_url('webcapture_dashboard', upload_short_id) }}" class="btn btn-primary">Web Capture</a>
+    <a href="{{ reverse_url('webcapture_index', upload_short_id) }}" class="btn btn-primary">Web Capture</a>
   </div>
 </div>
 

reproserver/web/templates/webcapture/crawl_results.html

@@ -6,7 +6,7 @@
   <ol class="breadcrumb">
     <li class="breadcrumb-item"><a href="{{ reverse_url('index') }}">Home</a></li>
     <li class="breadcrumb-item"><a href="{{ experiment_url }}">Package {{ run.upload.filename | truncate(60) }}</a></li>
-    <li class="breadcrumb-item"><a href="{{ reverse_url('webcapture_dashboard', upload_short_id) }}">Web capture</a></li>
+    <li class="breadcrumb-item"><a href="{{ reverse_url('webcapture_index', upload_short_id) }}">Web capture</a></li>
     <li class="breadcrumb-item active" aria-current="page">Automated capture</li>
   </ol>
 </nav>
@@ -32,7 +32,7 @@ <h1>Automated capture</h1>
 </div>
 
   {% if wacz %}
-<p>The capture was successful, <a href="{{ reverse_url('webcapture_dashboard', run.upload.short_id, wacz=wacz, hostname=hostname, port_number=port_number) }}">click here to return to the dashboard</a>.</p>
+<p>The capture was successful, <a href="{{ reverse_url('webcapture_done', run.upload.short_id, wacz, hostname=hostname, port_number=port_number) }}">click here to continue</a>.</p>
   {% else %}
 <p>The capture failed to produce a WACZ file!</p>
   {% endif %}

reproserver/web/templates/webcapture/done.html

@@ -0,0 +1,41 @@
+{% extends "base.html" %}
+
+{% block content %}
+
+<nav aria-label="breadcrumb">
+  <ol class="breadcrumb">
+    <li class="breadcrumb-item"><a href="{{ reverse_url('index') }}">Home</a></li>
+    <li class="breadcrumb-item"><a href="{{ experiment_url }}">Package {{ filename | truncate(60) }}</a></li>
+    <li class="breadcrumb-item" aria-current="page">Web capture</li>
+  </ol>
+</nav>
+
+<h1>Web Capture complete</h1>
+
+<p>You have completed the web capture. A WACZ file has been created containing the remote assets referenced by your application (CSS, JS, etc). This will allow it to keep working correctly in web browsers if those remote sources go away.</p>
+
+<div class="row">
+  <p class="col-sm-2">RPZ File:</p>
+  <p class="col-sm-10">{{ filename }}, {{ filesize | human_size }}</p>
+</div>
+
+<div class="row">
+  <p class="col-sm-2">WACZ FILE:</p>
+  <p class="col-sm-10">
+    {{ wacz.filesize | human_size }}
+  </p>
+</div>
+
+<div class="mb-3">
+  <form method="POST">
+    {{ xsrf_form_html() }}
+    <input type="hidden" name="port_number" value="{{ port_number }}">
+    <input type="hidden" name="hostname" value="{{ hostname }}">
+    <input type="submit" formaction="{{ reverse_url('webcapture_download', upload_short_id, wacz=wacz.hash) }}" class="btn btn-primary" value="Download finished RPZ">
+    <input type="submit" formaction="{{ reverse_url('webcapture_preview', upload_short_id, wacz=wacz.hash) }}" class="btn btn-secondary" value="Preview the result">
+  </form>
+</div>
+
+<p>If you want, you can <a href="{{ reverse_url('webcapture_index', upload_short_id, hostname=hostname, port_number=port_number) }}">start a new web capture</a>. This will discard the current WACZ file.</p>
+
+{% endblock content %}

reproserver/web/templates/webcapture/dashboard.html → reproserver/web/templates/webcapture/index.html

@@ -16,23 +16,29 @@ <h1>Web Capture</h1>
 
 <p>After completing this process, you will be able to download a new RPZ file with an embedded WACZ archive.</p>
 
-<h2 class="mb-3">Info about your files</h2>
-
-<div class="mb-3 row">
-  <p class="col-sm-2">RPZ File:</p>
-  <p class="col-sm-10">{{ filename }}, {{ filesize | human_size }}</p>
+<div class="row">
+  <p class="col-lg-2 col-sm-4">Input RPZ File:</p>
+  <p class="col-lg-10 col-sm-8">{{ filename }}, {{ filesize | human_size }}</p>
 </div>
-<div class="mb-3 row">
-  <p class="col-sm-2">WACZ FILE:</p>
-  <p class="col-sm-10">
-    {% if wacz %}
-    present, {{ wacz.filesize | human_size }}
-    {% else %}
-    none
-    {% endif %}
+
+{% if wacz %}
+<div class="row">
+  <p class="col-lg-2 col-sm-4">Detected WACZ FILE:</p>
+  <p class="col-lg-10 col-sm-8">
+    {{ wacz.filesize | human_size }}
   </p>
 </div>
 
+<div class="alert alert-primary d-flex align-items-center" role="alert">
+  <svg viewBox="0 0 16 16" class="flex-shrink-0 me-2" role="img" style="width: 1em; height: 1em;" aria-label="Warning:">
+    <path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
+  </svg>
+  <div>
+    Captured web content is already present. If you proceed, it will be replaced with a new WACZ file.
+  </div>
+</div>
+{% endif %}
+
 <h2 class="mb-3">Edit parameters for web capture</h2>
 
 <form method="POST">
@@ -87,19 +93,6 @@ <h2>Upload existing file</h2>
     </div>
   </div>
   <p>After the capture is complete, you will be redirected to this page. From there, get a preview of the replayed app, or download the new RPZ file (with the WACZ file in it).</p>
-  {% if wacz %}
-  <div class="row justify-content-center">
-    <div class="col-md-4">
-      <h2>Get the finished RPZ bundle</h2>
-      <div class="mb-3">
-        <input type="submit" formaction="{{ reverse_url('webcapture_download', upload_short_id, wacz=wacz.hash) }}" class="btn btn-primary" value="Download finished RPZ">
-      </div>
-      <div class="mb-3">
-        <input type="submit" formaction="{{ reverse_url('webcapture_preview', upload_short_id, wacz=wacz.hash) }}" class="btn btn-secondary" value="Preview the result">
-      </div>
-    </div>
-  </div>
-  {% endif %}
 </form>
 
 <script>

reproserver/web/templates/webcapture/record.html

@@ -10,7 +10,7 @@
   <ol class="breadcrumb">
     <li class="breadcrumb-item"><a href="{{ reverse_url('index') }}">Home</a></li>
     <li class="breadcrumb-item"><a href="{{ experiment_url }}">Package {{ run.upload.filename | truncate(60) }}</a></li>
-    <li class="breadcrumb-item"><a href="{{ reverse_url('webcapture_dashboard', upload_short_id) }}">Web capture</a></li>
+    <li class="breadcrumb-item"><a href="{{ reverse_url('webcapture_index', upload_short_id) }}">Web capture</a></li>
     <li class="breadcrumb-item active" aria-current="page">Live record</li>
   </ol>
 </nav>

reproserver/web/templates/webcapture/upload_wacz.html

@@ -6,7 +6,7 @@
   <ol class="breadcrumb">
     <li class="breadcrumb-item"><a href="{{ reverse_url('index') }}">Home</a></li>
     <li class="breadcrumb-item"><a href="{{ experiment_url }}">Package {{ upload.filename | truncate(60) }}</a></li>
-    <li class="breadcrumb-item"><a href="{{ reverse_url('webcapture_dashboard', upload_short_id) }}">Web capture</a></li>
+    <li class="breadcrumb-item"><a href="{{ reverse_url('webcapture_index', upload_short_id) }}">Web capture</a></li>
     <li class="breadcrumb-item active" aria-current="page">Upload a WACZ file</li>
   </ol>
 </nav>

reproserver/web/templates/webcapture/upload_wacz_bad.html

@@ -6,7 +6,7 @@
   <ol class="breadcrumb">
     <li class="breadcrumb-item"><a href="{{ reverse_url('index') }}">Home</a></li>
     <li class="breadcrumb-item"><a href="{{ experiment_url }}">Package {{ upload.filename | truncate(60) }}</a></li>
-    <li class="breadcrumb-item"><a href="{{ reverse_url('webcapture_dashboard', upload.short_id) }}">Web capture</a></li>
+    <li class="breadcrumb-item"><a href="{{ reverse_url('webcapture_index', upload.short_id) }}">Web capture</a></li>
     <li class="breadcrumb-item active" aria-current="page">Upload a WACZ file</li>
   </ol>
 </nav>

reproserver/web/webcapture.py

@@ -20,8 +20,8 @@
 logger = logging.getLogger(__name__)
 
 
-class Dashboard(BaseHandler):
-    @PROM_REQUESTS.sync('webcapture_dashboard')
+class Index(BaseHandler):
+    @PROM_REQUESTS.sync('webcapture_index')
     def get(self, upload_short_id):
         # Decode info from URL
         try:
@@ -30,17 +30,18 @@ def get(self, upload_short_id):
             self.set_status(404)
             return self.render('setup_notfound.html')
 
-        wacz_hash = self.get_query_argument('wacz', None)
-
         hostname = self.get_query_argument('hostname', '')
-        port_number = self.get_query_argument('port_number', '3000')
+        port_number = self.get_query_argument('port_number', '') or '3000'
         try:
             port_number = int(port_number, 10)
             if not (1 <= port_number <= 65535):
                 raise OverflowError
         except (ValueError, OverflowError):
             raise HTTPError(400, "Wrong port number")
 
+        if hostname == f'localhost:{port_number}':
+            hostname = ''
+
         # Look up the experiment in database
         upload = (
             self.db.query(database.Upload)
@@ -55,7 +56,14 @@ def get(self, upload_short_id):
             self.set_status(404)
             return self.render('setup_notfound.html')
 
-        if wacz_hash:
+        # Look for web extension
+        extensions = {
+            extension.name: json.loads(extension.data)
+            for extension in upload.experiment.extensions
+        }
+        if 'web1' in extensions:
+            wacz_hash = extensions['web1']['filehash']
+
             try:
                 meta = self.application.object_store.get_file_metadata(
                     'web1',
@@ -76,25 +84,10 @@ def get(self, upload_short_id):
                 )
             }
         else:
-            # Look for web extension
-            extensions = {
-                extension.name: json.loads(extension.data)
-                for extension in upload.experiment.extensions
-            }
-            if 'web1' in extensions:
-                wacz_hash = extensions['web1']['filehash']
-                return self.redirect(
-                    self.reverse_url(
-                        'webcapture_dashboard',
-                        upload_short_id,
-                        wacz=wacz_hash,
-                    ),
-                )
-
             wacz = None
 
         return self.render(
-            'webcapture/dashboard.html',
+            'webcapture/index.html',
             filename=upload.filename,
             filesize=upload.experiment.size,
             experiment_url=self.url_for_upload(upload),
@@ -105,6 +98,62 @@ def get(self, upload_short_id):
         )
 
 
+class Done(BaseHandler):
+    @PROM_REQUESTS.sync('webcapture_done')
+    def get(self, upload_short_id, wacz_hash):
+        # Decode info from URL
+        try:
+            upload_id = database.Upload.decode_id(upload_short_id)
+        except ValueError:
+            self.set_status(404)
+            return self.render('setup_notfound.html')
+
+        # Look up the experiment in database
+        upload = (
+            self.db.query(database.Upload)
+            .options(
+                joinedload(database.Upload.experiment).joinedload(
+                    database.Experiment.extensions,
+                )
+            )
+            .get(upload_id)
+        )
+        if upload is None:
+            self.set_status(404)
+            return self.render('setup_notfound.html')
+
+        try:
+            meta = self.application.object_store.get_file_metadata(
+                'web1',
+                wacz_hash + '.wacz',
+            )
+        except KeyError:
+            self.set_status(404)
+            return self.render('setup_notfound.html')
+
+        wacz = {
+            'hash': wacz_hash,
+            'filesize': meta['size'],
+            'url': self.application.object_store.presigned_serve_url(
+                'web1',
+                wacz_hash + '.wacz',
+                'archive.wacz',
+                'application/zip',
+            )
+        }
+
+        return self.render(
+            'webcapture/done.html',
+            filename=upload.filename,
+            filesize=upload.experiment.size,
+            experiment_url=self.url_for_upload(upload),
+            upload_short_id=upload.short_id,
+            wacz=wacz,
+            hostname=self.get_query_argument('hostname', ''),
+            port_number=self.get_query_argument('port_number', ''),
+        )
+
+
 class Preview(BaseHandler):
     @PROM_REQUESTS.sync('webcapture_preview')
     def post(self, upload_short_id):
@@ -466,6 +515,8 @@ def get(self, upload_short_id, run_short_id):
 
 
 class CrawlStatusWebsocket(WebSocketHandler, BaseHandler):
+    upstream_ws = None
+
     def get(self, upload_short_id, run_short_id):
         # Decode info from URL
         try:
@@ -517,7 +568,8 @@ def on_upstream_message(self, message):
             return self.write_message(message, isinstance(message, bytes))
 
     def on_ws_connection_close(self, close_code=None, close_reason=None):
-        self.upstream_ws.close(close_code, close_reason)
+        if self.upstream_ws is not None:
+            self.upstream_ws.close(close_code, close_reason)
 
 
 class UploadWacz(BaseHandler):
@@ -634,9 +686,9 @@ async def post(self, upload_short_id):
             self.db.commit()
 
         redirect_url = self.reverse_url(
-            'webcapture_dashboard',
+            'webcapture_done',
             upload_short_id,
-            wacz=wacz_hash,
+            wacz_hash,
             hostname=hostname,
             port_number=port_number,
         )