fixing bug that error spec features need to be scaled

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

app/main.py

@@ -4,7 +4,12 @@
 from fastapi.templating import Jinja2Templates
 
 import logging
-from .library.helpers import load_errors_lookup, load_errors_specs_lookup, load_good_specs, openfile
+from .library.helpers import (
+    load_errors_lookup,
+    load_errors_specs_lookup,
+    load_good_specs,
+    openfile,
+)
 from app.routers import errors
 
 

app/routers/errors.py

@@ -28,7 +28,14 @@ def find_similar_working_specs(request: Request, error_spec_hash: str, cluster_i
         request.app.root, error_spec_hash, spectype="errors"
     )
     error_spec_features = dict.fromkeys(error_spec_features, 1)
-    good_spec_features, good_spec_uid, score = find_single_nearest_good_spec(
+
+    # Here we replace error_spec_features with the normalized / scaled version
+    (
+        good_spec_features,
+        error_spec_features,
+        good_spec_uid,
+        score,
+    ) = find_single_nearest_good_spec(
         request.app.root, error_spec_features, cluster_id, request.app.good_specs
     )
     return spec_comparison_view(
@@ -85,6 +92,7 @@ def spec_comparison_view(
     # In the error spec but not the good spec
     # Features will overlap, but having a value == 0 is akin to not present
     error_not_good = {}
+
     for feat, value in error_spec_features.items():
         if feat in good_spec_features:
             good_value = good_spec_features[feat]
@@ -109,8 +117,9 @@ def spec_comparison_view(
             "request": request,
             "cluster_id": cluster_id,
             "overlap": overlap,
-            "good_spec": json.dumps(good_spec),
+            "good_spec": json.dumps(good_spec, indent=4),
             "good_spec_uid": good_spec_uid,
+            "error_not_good": error_not_good,
             "good_not_error": good_not_error,
             "comptype": comptype,
         },

app/routers/helpers.py

@@ -69,6 +69,7 @@ def find_single_nearest_good_spec(root, error_spec_features, cluster_id, good_sp
     feat_normalized = np.nan_to_num(
         feat_weight_df.to_numpy() / feat_weight_df.to_numpy().sum(axis=1)
     )
+
     # Weight the features of error spec
     feat_weighted = feat_df.to_numpy() * feat_normalized
 
@@ -77,9 +78,16 @@ def find_single_nearest_good_spec(root, error_spec_features, cluster_id, good_sp
     good_spec_score = np.matmul(good_specs.to_numpy(), np.transpose(feat_weighted))
     good_spec_features = good_specs.iloc[np.argmax(good_spec_score)].to_dict()
     good_spec_uid = good_specs.iloc[np.argmax(good_spec_score)].name
+    bad_spec_features = pd.DataFrame(feat_weighted[0])
+    bad_spec_features.index = feat_weight_df.columns
 
     # Return the uid for the closest good spec and the score
-    return good_spec_features, good_spec_uid, np.max(good_spec_score)
+    return (
+        good_spec_features,
+        bad_spec_features.to_dict()[0],
+        good_spec_uid,
+        np.max(good_spec_score),
+    )
 
 
 def find_cluster_nearest_good_spec(feat_weights, good_specs):

templates/analysis/diff.html

@@ -43,16 +43,29 @@ <h2 class="accordion-header" id="heading-{{ good_spec_uid }}">
 <div class="row">
   <div class="col-md-12">
    <h2>Overlapping Working and Error Spec Features</h2>
+   {% if overlap %}
    <p>This shows overlapping features for error cluster {{ cluster_id }} and the closest working spec {{ good_spec_uid }}</p>
-   <div style="height:700px"><canvas id="comparison"></canvas></div>
+   <div style="height:700px"><canvas id="comparison"></canvas></div>{% else %}
+    <p class="alert alert-info">There are no overlapping, non-zero features on either side.</p>
+   {% endif %}
 
    {% if good_not_error %}
-   <h2>Good features not present in error cluster features</h2>
+   <h2>Good features not present in error {% if comptype =="cluster"%}cluster{% endif %} features</h2>
    <p>The following features are present in the working spec (at a non-zero value) and not in the error cluster</p>
    <ul>
      {% for k, v in good_not_error.items() %}<li><strong>{{ k }}</strong>: {{ v }}</li>
     {% endfor %}</ul>
    {% endif %}
+
+   {% if error_not_good %}
+   <hr>
+   <h2>Error features not present in good features</h2>
+   <p>The following features are present for the error case but not the working case.r</p>
+   <ul>
+     {% for k, v in error_not_good.items() %}<li><strong>{{ k }}</strong>: {{ v }}</li>
+    {% endfor %}</ul>
+   {% endif %}
+
   </div>
 </div>
 {% endblock %}
@@ -61,6 +74,7 @@ <h2>Good features not present in error cluster features</h2>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.23.0/prism.min.js"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.5.0/Chart.min.js"></script>
 <script>
+{% if overlap %}
 function make_plot() {
 var data = {
   labels: [{% for feature in overlap %}"{{ feature[0] }}"{% if loop.last %}{% else %},{% endif %}{% endfor %}],
@@ -114,6 +128,6 @@ <h2>Good features not present in error cluster features</h2>
 });
 }
 
-make_plot()
+make_plot(){% endif %}
 </script>
 {% endblock %}