Correctness checking for security_barrier_camera_demo w/ 1 network multi channels with inputting 1 image

demos/common/cpp/utils/src/args_helper.cpp

@@ -81,7 +81,7 @@ std::vector<std::string> parseDevices(const std::string& device_string) {
     const std::string::size_type colon_position = device_string.find(":");
     if (colon_position != std::string::npos) {
         std::string device_type = device_string.substr(0, colon_position);
-        if (device_type == "HETERO" || device_type == "MULTI") {
+        if (device_type == "HETERO" || device_type == "MULTI" || device_type == "AUTO") {
             std::string comma_separated_devices = device_string.substr(colon_position + 1);
             std::vector<std::string> devices = split(comma_separated_devices, ',');
             for (auto& device : devices)

demos/security_barrier_camera_demo/cpp/main.cpp

@@ -220,8 +220,8 @@ class ClassifiersAggregator {
         std::mutex& printMutex = static_cast<ReborningVideoFrame*>(sharedVideoFrame.get())->context.classifiersAggregatorPrintMutex;
         printMutex.lock();
         if (FLAGS_r && !rawDetections.empty()) {
-            slog::debug << "Frame #: " << sharedVideoFrame->frameId << slog::endl;
-            slog::debug << rawDetections;
+            for (const std::string& rawDetection : rawDetections)
+                slog::debug << rawDetection << slog::endl;
             // destructor assures that none uses the container
             for (const std::string& rawAttribute : rawAttributes.container) {
                 slog::debug << rawAttribute << slog::endl;
@@ -292,6 +292,10 @@ ReborningVideoFrame::~ReborningVideoFrame() {
         context.videoFramesContext.lastFrameIdsMutexes[sourceID].lock();
         const auto frameId = ++context.videoFramesContext.lastframeIds[sourceID];
         context.videoFramesContext.lastFrameIdsMutexes[sourceID].unlock();
+
+        // Stop reborning when frame ID reached to input queue size
+        if (!context.isVideo && frameId >= FLAGS_n_iqs)
+            return;
         std::shared_ptr<ReborningVideoFrame> reborn = std::make_shared<ReborningVideoFrame>(context, sourceID, frameId, frame);
         worker->push(std::make_shared<Reader>(reborn));
     } catch (const std::bad_weak_ptr&) {}
@@ -379,7 +383,9 @@ void Drawer::process() {
         } else {
             if (!context.isVideo) {
                 try {
-                    std::shared_ptr<Worker>(context.drawersContext.drawersWorker)->stop();
+                    // Exit only when inferences on all of frames are finished.
+                    if (context.frameCounter >= FLAGS_n_iqs * context.readersContext.inputChannels.size())
+                        std::shared_ptr<Worker>(context.drawersContext.drawersWorker)->stop();
                 }
                 catch (const std::bad_weak_ptr&) {}
             }
@@ -428,7 +434,8 @@ bool DetectionsProcessor::isReady() {
     if (requireGettingNumberOfDetections) {
         classifiersAggregator = std::make_shared<ClassifiersAggregator>(sharedVideoFrame);
         std::list<Detector::Result> results;
-        results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections);
+        results = context.inferTasksContext.detector.getResults(*inferRequest, sharedVideoFrame->sourceID, sharedVideoFrame->frameId, sharedVideoFrame->frame.size(), classifiersAggregator->rawDetections);
+
         for (Detector::Result result : results) {
             switch (result.label) {
                 case 1:

demos/security_barrier_camera_demo/cpp/net_wrappers.hpp

@@ -121,9 +121,8 @@ class Detector {
         }
     }
 
-    std::list<Result> getResults(ov::InferRequest& inferRequest, cv::Size upscale, std::vector<std::string>& rawResults) {
-        // there is no big difference if InferReq of detector from another device is passed
-        // because the processing is the same for the same topology
+    std::list<Result> getResults(ov::InferRequest& inferRequest, const int64_t channelID, const int64_t frameID, cv::Size upscale, std::vector<std::string>& rawResults) {
+        // there is no big difference if InferReq of detector from another device is passed because the processing is the same for the same topology
         std::list<Result> results;
         ov::Tensor output_tensor = inferRequest.get_tensor(m_detectorOutputName);
         const float* const detections = output_tensor.data<float>();
@@ -146,8 +145,9 @@ class Detector {
             rect.height = static_cast<int>(detections[i * objectSize + 6] * upscale.height) - rect.y;
             results.push_back(Result{label, confidence, rect});
             std::ostringstream rawResultsStream;
-            rawResultsStream << "[" << i << "," << label << "] element, prob = " << confidence
-                        << "    (" << rect.x << "," << rect.y << ")-(" << rect.width << "," << rect.height << ")";
+            rawResultsStream << "ChannelId:" << channelID << ",FrameId:" << frameID << ",";
+            rawResultsStream << "ObjId:" << i << "," << label << "," << confidence
+                        << "," << rect.x << "," << rect.y << "," << rect.width << "," << rect.height;
             rawResults.push_back(rawResultsStream.str());
         }
         return results;

demos/tests/cases.py

@@ -77,6 +77,10 @@ def parse_output(self, output, test_case, device):
         if self.parser:
             self.parser(output, test_case, device)
 
+    def check_difference(self):
+        if self.parser:
+            return self.parser.check_difference()
+
     def update_option(self, updated_options):
         for case in self.test_cases[:]:
             self.update_case(case, updated_options, with_replacement=True)
@@ -602,7 +606,8 @@ def single_option_cases(key, *args):
         TestCase(options={'-no_show': None,
             **MONITORS,
             '-i': DataDirectoryArg('vehicle-license-plate-detection-barrier')}),
-        TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0106')}),
+        # Update to this model in order to obtain the ROI data
+        TestCase(options={'-m': ModelArg('vehicle-license-plate-detection-barrier-0123')}),
         single_option_cases('-m_lpr',
             None,
             ModelArg('license-plate-recognition-barrier-0001'),

demos/tests/correctness_cases.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from copy import deepcopy
+from abc import ABC, abstractmethod
+
+from cases import BASE
+
+class CorrectnessCheckerBase(ABC):
+    def __init__(self, demo):
+        self.filename = demo.subdirectory.replace('/', '_') + '.log'
+        self.demo_name = demo.subdirectory.split('/')[0]
+        self.results = {}
+        self.case_index = {}
+
+    @abstractmethod
+    def __call__(self, output, test_case, device, execution_time=-1):
+        pass
+
+    @abstractmethod
+    def check_difference(self):
+        pass
+
+    def write_to_log(self, result, test_case, device):
+        with open(self.filename, 'w') as f:
+            print(self.results, file=f)
+
+
+class DemoSecurityBarrierCamera(CorrectnessCheckerBase):
+    def __call__(self, output, test_case, device, execution_time=0):
+        # Results format
+        #               {"device name":
+        #                   {"case index 0":
+        #                       {"channel id 0":
+        #                           {"frame id 0":
+        #                               {"object id 0":{"label:xx,prob:xx,x,y,width,hight"},
+        #                               {"object id 1":{"label:xx,prob:xx,x,y,width,hight"},
+        #                               .....................
+        #                               {"object id n":{"label:xx,prob:xx,x,y,width,hight"}
+        #                           },
+        #                           .....
+        #                           {"frame id n":
+        #                               .....
+        #                           }
+        #                       },
+        #                       .....
+        #                       {"channel id n":
+        #                       .....
+        #                       }
+        #                   {"case index n":
+        #                       .....
+        #                   }
+        #               }
+
+        # Generate case id for each device
+        if device not in self.case_index:
+            self.case_index[device] = 0
+
+        if device not in self.results:
+            self.results[device] = {}
+
+        case_index = self.case_index[device]
+        if case_index not in self.results[device]:
+            self.results[device][case_index] = {}
+
+        # Parsing the raw data
+        output = [i.rstrip() for i in output.split('\n') if "DEBUG" in i and "ChannelId" in i]
+        for item in output:
+            item = item[item.find('ChannelId'):].split(',')
+            # Channel ID
+            frame_results = {}
+            channel = item[0].split(':')[1]
+            if channel not in self.results[device][case_index]:
+                self.results[device][case_index][channel] = frame_results
+
+            # Frame ID
+            object_results = {}
+            frame = item[1].split(':')[1]
+            if frame not in self.results[device][case_index][channel]:
+                self.results[device][case_index][channel][frame] = object_results
+
+            # Object ID
+            label_prob_pos_results = []
+            objid = item[2].split(':')[1]
+            if objid not in self.results[device][case_index][channel][frame]:
+                self.results[device][case_index][channel][frame][objid] = label_prob_pos_results
+            self.results[device][case_index][channel][frame][objid] = item[3:]
+
+        self.case_index[device] += 1
+
+    def check_difference(self):
+        flag = True
+        devices_list = {"AUTO:GPU,CPU" : ["CPU", "GPU"],
+                        "AUTO:CPU" : ["CPU"],
+                        "AUTO:GPU" : ["GPU"],
+                        "MULTI:GPU,CPU" : ["CPU", "GPU"]}
+        err_msg = ''
+        for device in devices_list:
+            for target in devices_list[device]:
+                if device not in self.results or target not in self.results:
+                    flag = False
+                    err_msg += "\tMiss the results of device {} or device {}.\n".format(device, target)
+                if device in self.results and target in self.results:
+                    if self.results[device] != self.results[target]:
+                        flag = False
+                        err_msg += "\tInconsistent results between device {} and {} \n".format(device, target)
+                        # Show the detailed inconsistent results
+                        for case in self.results[target]:
+                            if self.results[device][case] != self.results[target][case]:
+                                err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(device, case))
+                                for channel in self.results[device][case]:
+                                    for frame in self.results[device][case][channel]:
+                                        err_msg += ("\t\t\tChannel {} - Frame {} : {}\n".format(channel, frame, self.results[device][case][channel][frame]))
+                                err_msg += ('\t\t---------------------------------------------------------\n')
+                                err_msg += ("\t\t---* Device: {} - Case: {} *----\n".format(target, case))
+                                for channel in self.results[target][case]:
+                                    for frame in self.results[target][case][channel]:
+                                        err_msg += ("\t\t\tChannel {} - Frame {} : {}\n".format(channel, frame, self.results[target][case][channel][frame]))
+                                err_msg += ('\t\t---------------------------------------------------------\n')
+        if not flag:
+            print("Correctness checking: Failure\n{}".format(err_msg))
+        return flag
+
+DEMOS = [
+    BASE['security_barrier_camera_demo/cpp']
+    .update_option({'-r': None, '-n_iqs': '1'})
+    .add_parser(DemoSecurityBarrierCamera)
+]

demos/tests/run_tests.py

@@ -50,6 +50,7 @@
 scopes = {
     'base': importlib.import_module('cases').DEMOS,
     'performance': importlib.import_module('performance_cases').DEMOS,
+    'correctness': importlib.import_module('correctness_cases').DEMOS,
 }
 
 
@@ -71,7 +72,7 @@ def parse_args():
         help='list of demos to run tests for (by default, every demo is tested). '
         'For testing demos of specific implementation pass one (or more) of the next values: cpp, cpp_gapi, python.')
     parser.add_argument('--scope', default='base',
-        help='The scenario for testing demos.', choices=('base', 'performance'))
+        help='The scenario for testing demos.', choices=('base', 'performance', 'correctness'))
     parser.add_argument('--mo', type=Path, metavar='MO.PY',
         help='Model Optimizer entry point script')
     parser.add_argument('--devices', default="CPU GPU",
@@ -243,6 +244,7 @@ def main():
             dl_dir = prepare_models(auto_tools_dir, args.downloader_cache_dir, args.mo, global_temp_dir, demos_to_test, args.precisions)
 
         num_failures = 0
+        correctness_failures = 0
 
         try:
             pythonpath = f"{os.environ['PYTHONPATH']}{os.pathsep}"
@@ -261,7 +263,6 @@ def main():
             print(header)
             print()
             demo.set_precisions(args.precisions, model_info)
-
             declared_model_names = set()
             for model_data in json.loads(subprocess.check_output(
                     [sys.executable, '--', str(auto_tools_dir / 'info_dumper.py'),
@@ -342,7 +343,6 @@ def option_to_args(key, value):
                             print(output)
                             failed_tests.append(test_descr + '\n' + exit_msg)
                             num_failures += 1
-                            execution_time = -1
 
                         if args.report_file:
                             collect_result(demo.subdirectory, device, case_model_names, execution_time, args.report_file)
@@ -351,9 +351,13 @@ def option_to_args(key, value):
                                 write_log(header, args.log_file)
                             write_log(test_descr, args.log_file)
                             write_log(output, args.log_file)
-
+            if args.scope == "correctness":
+                print("Demo {} correctness checking....".format(demo.subdirectory))
+                if not demo.check_difference():
+                    correctness_failures += 1
             print()
-
+    if args.scope == "correctness":
+        print("Correctness Failures: {}".format(correctness_failures))
     print("{} failures:".format(num_failures))
     for test in failed_tests:
         print(test)