adding view to see plot of features and listing of specs. When we hav…

…e working specs we can add the "find similar working" view

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Makefile

@@ -1,2 +1,2 @@
 all:
-	uvicorn app.main:app --host=0.0.0.0 --port=5000
+	uvicorn app.main:app --host=0.0.0.0 --port=5000 --reload

app-requirements.txt

@@ -6,4 +6,5 @@ aiofiles==0.8.0
 python-multipart==0.0.5
 jinja2==3.0.3
 Markdown==3.3.6
-pytest==6.2.5
+pytest==6.2.5
+pyaml

app/main.py

@@ -37,6 +37,10 @@ async def load_app_data(request: Request, call_next):
         request.app.error_clusters = load_errors_lookup(root)
     if not hasattr(request.app, "spec_errors"):
         request.app.spec_errors = load_errors_specs_lookup(root)
+
+    # Save the app root and app directory root (here)
+    app.here = here
+    app.root = root
     return await call_next(request)
 
 

app/routers/errors.py

@@ -2,22 +2,68 @@
 from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
 import os
+import yaml
+import json
 
 router = APIRouter()
 templates = Jinja2Templates(directory="templates/")
 
 
+def load_spec(root, digest):
+    """
+    Given a spec id, load from the data directory
+    """
+    spec_file = os.path.join(root, "data", "spec_files", "%s.yaml" % digest)
+    if os.path.exists(spec_file):
+        with open(spec_file, "r") as fd:
+            content = yaml.safe_load(fd.read())
+        return content
+
+
+def load_features(root, digest):
+    """
+    Given a spec id, load features for it
+    """
+    spec_file = os.path.join(
+        root, "data", "feature-ranking", "tfidf", "cluster-feats-%s.json" % digest
+    )
+    if os.path.exists(spec_file):
+        with open(spec_file, "r") as fd:
+            content = json.loads(fd.read())
+        return content
+
+
+@router.get("/analysis/features/{cluster_id}", response_class=HTMLResponse)
+def view_features(request: Request, cluster_id: int):
+    """
+    Clicking to see features for a cluster will show the ranked list.
+    """
+    features = load_features(request.app.root, cluster_id)
+
+    # Only return non-zero features and sort by value
+    if features:
+        features = {k: v for k, v in features.items() if v != 0.0}
+        features = sorted(features.items(), key=lambda x: x[1], reverse=True)
+    return templates.TemplateResponse(
+        "analysis/features.html",
+        context={"request": request, "cluster_id": cluster_id, "features": features},
+    )
+
+
 @router.get("/analysis/errors/{cluster_id}", response_class=HTMLResponse)
 def view_error(request: Request, cluster_id: int):
     """
     Clicking an error (assigned to a cluster) should show all the specs assigned.
     """
     # Find spec ids that match the cluster
-    specs = []
+    specs = {}
     for spec_hash, cid in request.app.spec_errors.items():
         if cid == cluster_id:
-            specs.append(spec_hash)
+            spec = load_spec(request.app.root, spec_hash)
+            if spec:
+                # Ensure we pretty print it
+                specs[spec_hash] = json.dumps(spec, indent=4)
     return templates.TemplateResponse(
-        "error-cluster.html",
+        "analysis/error-cluster.html",
         context={"request": request, "cluster_id": cluster_id, "specs": specs},
     )

static/css/style3.css

@@ -1,8 +1,15 @@
 /*
-    DEMO STYLE
     https://bootstrapious.com/p/bootstrap-sidebar
 */
 
+
+/* Buttons */
+
+.right-button {
+  float: right;
+  margin-right: 5px;
+}
+
 @import "https://fonts.googleapis.com/css?family=Poppins:300,400,500,600,700";
 body {
     font-family: 'Poppins', sans-serif;

templates/analysis/error-cluster.html

@@ -0,0 +1,49 @@
+{% extends "base.html" %}
+{% set active_page = "error-cluster" %}
+
+{% block title %}Error Cluster {{ cluster_id }}{% endblock %}
+{% block head %}        
+<link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.23.0/themes/prism-coy.min.css" rel="stylesheet" type="text/css">
+{{ super() }}
+
+{% endblock %}
+
+{% block page_content %}
+
+<div class="row">
+  <div class="col-md-4">
+    <h1 class="cap">Error Cluster {{ cluster_id }}</h1>
+     <p><strong>{{ specs | length }}</strong> specs are assigned to this cluster!</p>
+  </div>
+  <div class="col-md-8">
+    <a href="/analysis/features/{{ cluster_id }}" type="button" class="btn btn-primary right-button">View Features</a>
+    <a type="button" class="btn btn-primary right-button">Find Similar Working</a>
+  </div>
+</div>
+
+<div class="row"><div class="col-md-12">
+<div class="accordion" id="accordionSpecs">
+{% for spec_hash, spec in specs.items() %}<div class="accordion-item">
+    <h2 class="accordion-header" id="heading-{{ spec_hash }}">
+      <button class="accordion-button" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-{{ spec_hash }}" aria-expanded="true" aria-controls="collapse-{{ spec_hash }}">
+        <strong style="margin-right:5px">Spec #{{ loop.index }}</strong> {{ spec_hash }}
+      </button>
+    </h2>
+    <div id="collapse-{{ spec_hash }}" class="accordion-collapse collapse" aria-labelledby="heading" data-bs-parent="#accordionSpecs">
+      <div class="accordion-body">
+       <pre><code class="language-javascript">
+        {{ spec }}
+       </code></pre>
+      </div>
+    </div>
+  </div>
+{% endfor %}
+</div>
+</div></div> <!-- end row/col-->
+
+{% endblock %}
+
+{% block scripts %}
+{{ super() }}
+<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.23.0/prism.min.js"></script>
+{% endblock %}

templates/analysis/features.html

@@ -0,0 +1,80 @@
+{% extends "base.html" %}
+{% set active_page = "features" %}
+
+{% block title %}Features for Cluster {{ cluster_id }}{% endblock %}
+{% block head %}
+{{ super() }}
+
+{% endblock %}
+
+{% block page_content %}
+
+<div class="row">
+  <div class="col-md-4">
+    <h1 class="cap">Features for Cluster {{ cluster_id }}</h1>
+     {% if not features %}<p>This cluster does not have any features.</p>{% endif %}
+  </div>
+  <div class="col-md-8">
+    <a href="/analysis/errors/{{ cluster_id }}" type="button" class="btn btn-primary right-button">Back to Specs</a>
+    <a type="button" class="btn btn-primary right-button">Find Similar Working</a>
+  </div>
+</div>
+
+<div class="row"><div class="col-md-12">
+  <div style="height:700px"><canvas id="features"></canvas></div>
+</div></div> <!-- end row/col-->
+
+{% endblock %}
+
+{% block scripts %}
+{{ super() }}
+
+<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.5.0/Chart.min.js"></script>
+<script>
+function make_plot() {
+var data = {
+  labels: [{% for feature in features %}"{{ feature[0] }}"{% if loop.last %}{% else %},{% endif %}{% endfor %}],
+  datasets: [{
+    label: "Features for Cluster {{ cluster_id }}",
+    backgroundColor: "rgba(15, 58, 128, 0.47)",
+    borderColor: "rgb(4, 69, 174)",
+    borderWidth: 2,
+    hoverBackgroundColor: "rgb(15, 58, 128)",
+    hoverBorderColor: "rgb(102, 16, 242)",
+    data: [{% for feature in features %}{{ feature[1]}}{% if loop.last %}{% else %},{% endif %}{% endfor %}],
+  }]
+};
+
+var option = {
+  scales: {
+    yAxes: [{
+      stacked: true,
+      gridLines: {
+        display: true,
+        color: "rgba(15, 58, 128, 0.2)"
+      }
+    }],
+    xAxes: [{
+      gridLines: {
+        display: false
+      },
+      ticks: {
+        autoSkip: false
+      }
+    }]
+  }
+};
+
+Chart.Bar('features', {
+  options: option,
+  data: data,
+  options: {
+        responsive: true,
+        maintainAspectRatio: false
+    }
+});
+}
+
+make_plot()
+</script>
+{% endblock %}