Neural Insights improvements (#946)

Signed-off-by: bmyrcha <bartosz.myrcha@intel.com>
Signed-off-by: chensuyue <suyue.chen@intel.com>

.azure-pipelines/scripts/install_neural_insights.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo "Install Neural Insights ... "
+cd /neural-compressor
+python -m pip install --no-cache-dir -r neural_insights/requirements.txt
+python setup.py neural_insights bdist_wheel
+pip install dist/neural_insights*.whl
+pip list

.azure-pipelines/scripts/ut/coverage.file

@@ -14,6 +14,7 @@ omit =
  */neural_compressor/adaptor/tf_utils/graph_rewriter/generic/fuse_decomposed_in.py
  */neural_compressor/adaptor/tf_utils/quantize_graph/qdq/fuse_qdq_in.py
  */neural_compressor/adaptor/tf_utils/graph_rewriter/int8/freeze_value.py
+ */neural_compressor/template/*
 exclude_lines =
  pragma: no cover
  raise NotImplementedError

.azure-pipelines/scripts/ut/run_neural_insights.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+python -c "import neural_compressor as nc;print(nc.version.__version__)"
+echo "run neural insights ut..."
+
+# Install Neural Solution
+bash /neural-compressor/.azure-pipelines/scripts/install_neural_insights.sh
+
+# Install requirements for test
+cd /neural-compressor/neural_insights/test || exit 1
+if [ -f "requirements.txt" ]; then
+    n=0
+    until [ "$n" -ge 3 ]
+    do
+        python -m pip install --no-cache-dir -r requirements.txt && break
+        n=$((n+1))
+        sleep 5
+    done
+    pip list
+else
+    echo "Not found requirements.txt file."
+fi
+
+cd /neural-compressor/neural_insights || exit 1
+find ./test -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh
+
+LOG_DIR=/neural-compressor/log_dir
+mkdir -p ${LOG_DIR}
+ut_log_name=${LOG_DIR}/ut_neural_insights.log
+
+echo "cat run.sh..."
+cat run.sh | tee ${ut_log_name}
+echo "------UT start-------"
+bash run.sh 2>&1 | tee -a ${ut_log_name}
+echo "------UT end -------"
+
+if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "core dumped" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then
+    echo "Find errors in UT test, please check the output..."
+    exit 1
+fi
+echo "UT finished successfully! "

.azure-pipelines/ut-neural-insights.yaml

@@ -0,0 +1,31 @@
+trigger: none
+
+pr:
+  autoCancel: true
+  drafts: false
+  branches:
+    include:
+      - master
+  paths:
+    include:
+      - neural_insights
+      - setup.py
+
+pool: ICX-16C
+
+variables:
+  UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
+
+stages:
+  - stage:
+    displayName: Unit Test for Neural Insights
+    jobs:
+      - job:
+        steps:
+          - template: template/ut-template.yml
+            parameters:
+              dockerConfigName: 'commonDockerConfig'
+              utScriptFileName: 'run_neural_insights'
+              uploadPath: $(UPLOAD_PATH)
+              utArtifact: 'ut-neural-insights'
+              utContainerName: "utTest-nInsights"

neural_compressor/strategy/strategy.py

@@ -1771,13 +1771,9 @@ def _diagnosis(self, tune_cfg):
                 "Input model mean": "input_stats.mean",
                 "Input model standard deviation": "input_stats.std",
                 "Input model variance": "input_stats.var",
-                "Optimized model min": "optimized_stats.min",
-                "Optimized model max": "optimized_stats.max",
-                "Optimized model mean": "optimized_stats.mean",
-                "Optimized model standard deviation": "optimized_stats.std",
-                "Optimized model variance": "optimized_stats.var",
             },
-            table_entries=sorted_weights_details
+            table_entries=sorted_weights_details,
+            precision=5,
         )
         logger.info("For more details execute quantization with Neural Insights GUI.")
 

neural_compressor/utils/utility.py

@@ -714,6 +714,7 @@ def print_table(
         output_handler=logger.info,
         title: Optional[str] = None,
         insert_newlines=False,
+        precision: Optional[int] = None,
 ) -> None:
     """Print table with prettytable.
 
@@ -723,11 +724,15 @@ def print_table(
              to object's attribute.
         table_entries (list): List of objects to be included in the table.
         output_handler (func, optional): Output handler function.
+        title (str): Title for the table
+        insert_newlines (bool): Add newlines to each row
+        precision (int): Number of digits after the decimal point
 
     Returns:
         None
     """
     from functools import reduce
+    import numpy as np
     table = pt.PrettyTable(min_table_width=40)
     if title is not None:
         table.title = title
@@ -739,6 +744,11 @@ def print_table(
                 table_row.append(entry.get(attribute))
             else:
                 value = reduce(getattr, [entry] + attribute.split("."))
+                if (isinstance(value, np.floating) or isinstance(value, float)) and isinstance(precision, int):
+                    if "e" in str(value):
+                        value = f'{value:.{precision}e}'
+                    else:
+                         value = round(value, precision)
                 table_row.append(value)
         table.add_row(table_row)
     lines = table.get_string().split('\n')
@@ -803,14 +813,14 @@ def get_weights_details(workload_location: str) -> list:
 
         if isinstance(input_model_op_tensors, dict):
             tensors_data = zip(input_model_op_tensors.items(), optimized_model_op_tensors.items())
-            for (input_op_name, input_op_values), (optimized_op_name, optimized_op_values) in tensors_data:
-                if input_op_values.shape != optimized_op_values.shape:
+            for (_, input_op_tensor_values), (_, optimized_op_tensor_values) in tensors_data:
+                if input_op_tensor_values.shape != optimized_op_tensor_values.shape:
                     continue
 
                 weights_entry = WeightsDetails(
-                    input_op_name,
-                    input_op_values,
-                    optimized_op_values,
+                    op_name,
+                    input_op_tensor_values,
+                    optimized_op_tensor_values,
                 )
                 weights_details.append(weights_entry)
     return weights_details
@@ -961,6 +971,7 @@ def print_op_list(workload_location: str):
             "Activation max": "activation_max",
         },
         table_entries=sorted_op_list,
+        precision=5,
     )
 
     activations_table_file = os.path.join(

neural_compressor/utils/weights_details.py

@@ -19,6 +19,7 @@
 
 from neural_compressor.utils.utility import mse_metric_gap
 
+PRECISION = 5
 
 class WeightsDetails:
     """Weights details class."""
@@ -68,9 +69,9 @@ def serialize(self) -> Dict[str, Any]:
             Dictionary with serialized WeightsStatistics object
         """
         return {
-            "Min weight": self.min,
-            "Max weight": self.max,
-            "Mean": self.mean,
-            "Standard deviation": self.std,
-            "Variance": self.var,
+            "Min weight": round(self.min, PRECISION),
+            "Max weight": round(self.max, PRECISION),
+            "Mean": round(self.mean, PRECISION),
+            "Standard deviation": round(self.std, PRECISION),
+            "Variance": round(self.var, PRECISION),
         }

neural_insights/__init__.py

@@ -13,6 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# flake8: noqa
 """The neural_insights package contains all components required for running Neural Insights."""
 
 from neural_insights.ni import NeuralInsights

neural_insights/bin/neural_insights.py

@@ -20,9 +20,10 @@
 gevent.monkey.patch_all()
 
 
-def exec():
+def execute():
     from neural_insights.main import main
     main()
 
-if __name__ == '__main__':
-    exec()
+
+if __name__ == "__main__":
+    execute()

neural_insights/components/diagnosis/diagnosis.py

@@ -102,9 +102,6 @@ def get_op_list(self) -> List[dict]:
 
     def get_weights_details(self, inspect_type: str) -> List[WeightsDetails]:
         """Get weights details for model."""
-        check_module("numpy")
-        import numpy as np
-
         weights_details = []
 
         minmax_file_path = os.path.join(
@@ -132,7 +129,8 @@ def get_weights_details(self, inspect_type: str) -> List[WeightsDetails]:
                 continue
 
             if isinstance(input_model_op_tensors, dict):
-                for (input_op_name, input_op_values), (optimized_op_name, optimized_op_values) in zip(input_model_op_tensors.items(), optimized_model_op_tensors.items()):
+                for (input_op_name, input_op_values), (optimized_op_name, optimized_op_values) in\
+                        zip(input_model_op_tensors.items(), optimized_model_op_tensors.items()):
                     if input_op_values.ndim != 4 or optimized_op_values.ndim != 4:
                         continue
 
@@ -151,7 +149,6 @@ def calculate_mse(
         optimized_model_tensors: dict,
     ) -> Optional[float]:
         """Calculate MSE for specified tensors."""
-
         input_model_op_data = input_model_tensors.get(op_name, None)
         optimized_model_op_data = optimized_model_tensors.get(op_name, None)
 
@@ -255,7 +252,7 @@ def get_weights_data(self, op_name: str, channel_normalization=True) -> list:
         tensors = self.get_tensors_info(model_type="optimized").get("weight", None)
         if tensors is None:
             raise ClientErrorException(
-                f"Could not get tensor information to display activations.",
+                "Could not get tensor information to display activations.",
             )
 
         op_tensors: Optional[dict] = tensors.get(op_name, None)
@@ -281,12 +278,9 @@ def get_weights_data(self, op_name: str, channel_normalization=True) -> list:
 
             for tensor in tensor_data:
                 if channel_normalization:
-                    tensor = 255 * (tensor-np.min(tensor))/(np.max(tensor) - np.min(tensor))
+                    tensor = 255 * (tensor - np.min(tensor)) / (np.max(tensor) - np.min(tensor))
                 img = Image.fromarray(tensor)
                 img = img.convert("L")
                 img.show()
-                # filename = f"tf/tensor_activations/{tensor_name}/{idx}.jpg"
-                # os.makedirs(os.path.dirname(filename), exist_ok=True)
-                # img.save(filename)
                 weights.append(tensor.tolist())
         return weights

neural_insights/components/diagnosis/weights_details.py

@@ -73,8 +73,3 @@ def serialize(
             "Standard deviation": self.std,
             "Variance": self.var,
         }
-
-
-
-
-

neural_insights/components/graph/collapser.py

@@ -131,4 +131,4 @@ def _unprepare_group_name(self, name: str) -> str:
         if not name.startswith(self.GROUP_NAME_PREFIX):
             return name
 
-        return name[len(self.GROUP_NAME_PREFIX) :]
+        return name[len(self.GROUP_NAME_PREFIX):]

neural_insights/components/workload_manager/quantization_workload.py

@@ -6,7 +6,8 @@
 
 class QuantizationWorkload(Workload):
 
-    def __init__(self,  data: Optional[dict] = None):
+    def __init__(self, data: Optional[dict] = None):
+        """Initialize Quantization Workload."""
         super().__init__(data)
         if data is None:
             data = {}
@@ -48,8 +49,8 @@ def optimized_accuracy(self) -> Optional[float]:
 
     @property
     def ratio(self) -> Optional[float]:
-        """
-        Get accuracy ratio.
+        """Get accuracy ratio.
+
         Returns: accuracy ratio if baseline and optimized accuracy are present
                  Otherwise returns None
         """

neural_insights/gui/src/App.scss

@@ -30,6 +30,7 @@ table.rounded {
   th.header {
     background: $accent-yellow;
     color: black;
+    vertical-align: middle;
   }
 
   td.header {

neural_insights/gui/src/components/Diagnosis/Diagnosis.js

@@ -24,6 +24,7 @@ import Warning from './../Warning/Warning';
 import Form from 'react-bootstrap/Form';
 import InputGroup from 'react-bootstrap/InputGroup';
 import Button from 'react-bootstrap/esm/Button';
+import Spinner from 'react-bootstrap/Spinner';
 
 function Diagnosis() {
   const [selectedNode, setSelectedNode] = useState(null);
@@ -142,25 +143,34 @@ class NodeSearch extends React.Component {
 function AccuracyResults({ selectedWorkload }) {
   return (
     <div className='data-panel'>
-      <table className='accuracy-table'>
-        <tbody>
-          <tr>
-            <td className="accuracy-title">Accuracy <br /> results</td>
-            <td>
-              <div className="accuracy-number">{selectedWorkload.accuracy_data.baseline_accuracy * 100}%</div>
-              <div className="accuracy-subtitle">FP32</div>
-            </td>
-            <td>
-              <div className="accuracy-number">{selectedWorkload.accuracy_data.optimized_accuracy * 100}%</div>
-              <div className="accuracy-subtitle">INT8</div>
-            </td>
-            <td>
-              <div className="accuracy-number">{Math.round(selectedWorkload.accuracy_data.ratio * 100) / 100}</div>
-              <div className="accuracy-subtitle">Ratio</div>
-            </td>
-          </tr>
-        </tbody>
-      </table>
+      {selectedWorkload.status === 'wip' &&
+        <p> Quantization is in progress.
+          <div className="spinner-container">
+            <Spinner className="spinner" animation="border" />
+          </div>
+        </p>
+      }
+      {selectedWorkload.status !== 'wip' &&
+        <table className='accuracy-table'>
+          <tbody>
+            <tr>
+              <td className="accuracy-title">Accuracy <br /> results</td>
+              <td>
+                <div className="accuracy-number">{selectedWorkload.accuracy_data.baseline_accuracy * 100}%</div>
+                <div className="accuracy-subtitle">FP32</div>
+              </td>
+              <td>
+                <div className="accuracy-number">{selectedWorkload.accuracy_data.optimized_accuracy * 100}%</div>
+                <div className="accuracy-subtitle">INT8</div>
+              </td>
+              <td>
+                <div className="accuracy-number">{(selectedWorkload.accuracy_data.ratio * 100).toPrecision(2)}%</div>
+                <div className="accuracy-subtitle">Ratio</div>
+              </td>
+            </tr>
+          </tbody>
+        </table>
+      }
     </div>
   )
 }

neural_insights/gui/src/components/Diagnosis/Diagnosis.scss

@@ -125,4 +125,13 @@
     width: 50vw;
     z-index: 100;
   }
+
+  .spinner-container {
+    width: 50px;
+    margin: auto;
+  }
+
+  .spinner {
+    color: #5B69FF;
+  }
 }