Enhanced transcoding integration tests with ffprobe report

apps/transcoding/ffmpeg/utils.py

@@ -4,7 +4,7 @@
 import os
 from pathlib import Path
 import shutil
-from typing import Optional
+from typing import List, Optional
 
 from apps.transcoding import common
 from apps.transcoding.common import ffmpegException
@@ -30,6 +30,7 @@ class Commands(enum.Enum):
     SPLIT = ('split', 'split-results.json')
     TRANSCODE = ('transcode', '')
     MERGE = ('merge', '')
+    COMPUTE_METRICS = ('compute-metrics', '')
 
 
 class StreamOperator:
@@ -191,3 +192,62 @@ def _specify_dir_mapping(output, temporary, resources, logs, work):
                                                     temporary=temporary,
                                                     resources=resources,
                                                     logs=logs, work=work)
+
+    def get_metadata(self,
+                     input_files: List[str],
+                     resources_dir: str,
+                     work_dir: str,
+                     output_dir: str) -> dict:
+
+        assert os.path.isdir(resources_dir)
+        assert all([
+            os.path.isfile(os.path.join(resources_dir, input_file))
+            for input_file in input_files
+        ])
+
+        try:
+            os.makedirs(output_dir, exist_ok=True)
+        except OSError:
+            raise ffmpegException(
+                "Failed to prepare directory structure for get_metadata")
+
+        metadata_requests = [{
+            'video': input_file,
+            'output': f'metadata-logs-{os.path.splitext(input_file)[0]}.json'
+        } for input_file in input_files]
+
+        extra_data = {
+            'entrypoint': FFMPEG_ENTRYPOINT,
+            'command': Commands.COMPUTE_METRICS.value[0],
+            'metrics_params': {
+                'metadata': metadata_requests,
+            },
+        }
+
+        dir_mapping = DockerTaskThread.specify_dir_mapping(
+            output=output_dir,
+            temporary=work_dir,
+            resources=resources_dir,
+            logs=work_dir,
+            work=work_dir)
+
+        logger.info('Obtaining video metadata.')
+        logger.debug('Command params: %s', extra_data)
+
+        job_result = self._do_job_in_container(dir_mapping, extra_data)
+        if 'data' not in job_result:
+            raise ffmpegException(
+                "Failed to obtain video metadata. "
+                "'data' not found in the returned JSON.")
+
+        if len(job_result['data']) < len(input_files):
+            raise ffmpegException(
+                "Failed to obtain video metadata. "
+                "Missing output for at least one input file.")
+
+        if len(job_result['data']) > len(input_files):
+            raise ffmpegException(
+                "Failed to obtain video metadata. Too many results.")
+
+        logger.info('Video metadata obtained successfully!')
+        return job_result

golem/testutils.py

@@ -40,6 +40,10 @@
 logger = logging.getLogger(__name__)
 
 
+class DockerTestJobFailure(Exception):
+    pass
+
+
 class TestTaskManager(TaskManager):
     def __init__(self, node, keys_auth, root_path, config_desc: ClientConfigDescriptor, tasks_dir="tasks",
                  task_persistence=False, apps_manager=AppsManager(), finished_cb=None):
@@ -482,7 +486,7 @@ def _run_test_job(self, task, root_dir, params):
         self._log_docker_logs(dtt)
 
         if dtt.error:
-            raise Exception(dtt.error_msg)
+            raise DockerTestJobFailure(dtt.error_msg)
 
         return dtt.result.get('data')
 

requirements-test.txt

@@ -13,6 +13,7 @@ idna==2.5
 isort==4.3.4
 lazy-object-proxy==1.3.1
 mccabe==0.6.1
+parameterized==0.7.0
 pluggy==0.6.0
 py==1.5.3
 pycodestyle==2.4.0

requirements-test_to-freeze.txt

@@ -4,6 +4,7 @@ factory-boy==2.9.2
 Faker==0.8.9
 flake8
 freezegun==0.3.11
+parameterized==0.7.0
 pylint==1.9.2
 pytest-cov
 pytest==3.3.1