Sb/time series checker

tools/accuracy_checker/accuracy_checker/adapters/README.md

@@ -412,3 +412,6 @@ AccuracyChecker supports following set of adapters:
   * `max_active` - max active paths for decoding (Optional, default `7000`).
   * `inverse_acoustic_scale` - inverse acoustic scale for lattice scaling (Optional, default `0`).
   * `word_insertion_penalty` - add word insertion penalty to the lattice. Penalties are negative log-probs, base e, and are added to the language model' part of the cost (Optional, `0`).
+* `quantiles_predictor` - converts output of Time Series Forecasting models to `TimeSeriesForecastingQuantilesPrediction`.
+  * `quantiles` - preds[i]->quantile[i] mapping.
+  * `output_name` - name of output node to convert.

tools/accuracy_checker/accuracy_checker/adapters/__init__.py

@@ -114,6 +114,8 @@
 from .noise_suppression import NoiseSuppressionAdapter
 from .dummy_adapters import GVADetectionAdapter, XML2DetectionAdapter, GVAClassificationAdapter
 
+from .time_series import QuantilesPredictorAdapter
+
 __all__ = [
     'Adapter',
     'AdapterField',
@@ -241,4 +243,5 @@
     'GVADetectionAdapter',
     'GVAClassificationAdapter',
 
+    'QuantilesPredictorAdapter'
 ]

tools/accuracy_checker/accuracy_checker/adapters/time_series.py

@@ -0,0 +1,47 @@
+"""
+Copyright (c) 2018-2021 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from .adapter import Adapter
+from ..representation import TimeSeriesForecastingQuantilesPrediction
+from ..config import StringField, DictField
+
+
+class QuantilesPredictorAdapter(Adapter):
+    __provider__ = 'quantiles_predictor'
+
+    @classmethod
+    def parameters(cls):
+        parameters = super().parameters()
+        parameters.update({
+            'quantiles': DictField(
+                allow_empty=False,
+                description="preds[i]->quantile[i] mapping."
+            ),
+            "output_name": StringField()
+        })
+        return parameters
+
+    def configure(self):
+        self.quantiles = self.get_value_from_config('quantiles')
+        self.output_name = str(self.get_value_from_config('output_name'))
+
+    def process(self, raw, identifiers, frame_meta):
+        raw_outputs = self._extract_predictions(raw, frame_meta)
+        output = raw_outputs[self.output_name]
+        preds = TimeSeriesForecastingQuantilesPrediction(identifiers[0])
+        for k, v in self.quantiles.items():
+            preds[k] = output[:, :, v]
+        return [preds]

tools/accuracy_checker/accuracy_checker/annotation_converters/README.md

@@ -521,6 +521,9 @@ The main difference between this converter and `super_resolution` in data organi
   * `ref_data_dir` - directory with reference ark files (Optional, if not provided `data_dir` will be used instead).
   * `vectors_mode` - allow usage each vector in utterance as independent data.
   * `ref_file_suffix` - suffix for search reference files (Optional, default `_kaldi_score`).
+* `electricity` - converts Electricity dataset to `TimeSeriesForecastingAnnotation`.
+  * `data_path_file` - Path to dataset file in .csv format.
+  * `num_encoder_steps` - The maximum number of historical timestamps that model use.
 * `yolo_labeling` - converts object detection dataset with annotation in YOLO labeling format to `DetectionAnnotation`.
   * `data_dir` - path to directory with annotation files in txt format and images.
   * `labels_file` - path to file with labels in txt format (optional).

tools/accuracy_checker/accuracy_checker/annotation_converters/__init__.py

@@ -103,6 +103,7 @@
 from .noise_suppression_dataset import NoiseSuppressionDatasetConverter
 from .vimeo90k_sr import Vimeo90KSuperResolutionDatasetConverter
 from .lmdb import LMDBConverter
+from .electricity_time_series_forecasting import ElectricityTimeSeriesForecastingConverter
 from .kaldi_speech_recognition_pipeline import KaldiSpeechRecognitionDataConverter, KaldiFeatureRegressionConverter
 from .yolo_labeling_converter import YOLOLabelingConverter
 
@@ -201,6 +202,7 @@
     'NoiseSuppressionDatasetConverter',
     'Vimeo90KSuperResolutionDatasetConverter',
     'LMDBConverter',
+    'ElectricityTimeSeriesForecastingConverter',
     'KaldiSpeechRecognitionDataConverter',
     'KaldiFeatureRegressionConverter',
     'ParametricImageProcessing',