Merge pull request #852 from Oren-i/master

Handle timeout errors when calling fact_extractor, related to #820

src/unpacker/unpack.py

@@ -36,6 +36,10 @@ def unpack(self, current_fo: FileObject):
         with TemporaryDirectory(prefix='fact_unpack_', dir=self.config['data-storage']['docker-mount-base-dir']) as tmp_dir:
             file_path = self._generate_local_file_path(current_fo)
             extracted_files = self.extract_files_from_file(file_path, tmp_dir)
+            if extracted_files is None:
+                self._store_unpacking_error_skip_info(current_fo)
+                return []
+
             extracted_file_objects = self.generate_and_store_file_objects(extracted_files, Path(tmp_dir) / 'files', current_fo)
             extracted_file_objects = self.remove_duplicates(extracted_file_objects, current_fo)
             self.add_included_files_to_object(extracted_file_objects, current_fo)
@@ -44,14 +48,21 @@ def unpack(self, current_fo: FileObject):
 
         return extracted_file_objects
 
+    @staticmethod
+    def _store_unpacking_error_skip_info(file_object: FileObject):
+        file_object.processed_analysis['unpacker'] = {
+            'plugin_used': 'None', 'number_of_unpacked_files': 0, 'plugin_version': '0.0', 'analysis_date': time(),
+            'info': 'Unpacking stopped because extractor raised a exception (possible timeout)',
+            'tags': {'extractor error': {'value': 'possible extractor timeout', 'color': TagColor.ORANGE, 'propagate': False}},
+        }
+
     @staticmethod
     def _store_unpacking_depth_skip_info(file_object: FileObject):
         file_object.processed_analysis['unpacker'] = {
             'plugin_used': 'None', 'number_of_unpacked_files': 0, 'plugin_version': '0.0', 'analysis_date': time(),
             'info': 'Unpacking stopped because maximum unpacking depth was reached',
+            'tags': {'depth reached': {'value': 'unpacking depth reached', 'color': TagColor.ORANGE, 'propagate': False}},
         }
-        tag_dict = {'unpacker': {'depth reached': {'value': 'unpacking depth reached', 'color': TagColor.ORANGE, 'propagate': False}}}
-        file_object.analysis_tags.update(tag_dict)
 
     def cleanup(self, tmp_dir):
         try:

src/unpacker/unpack_base.py

@@ -7,6 +7,7 @@
 from common_helper_files import safe_rglob
 from docker.types import Mount
 
+from requests import exceptions
 from helperFunctions.docker import run_docker_container
 
 
@@ -23,17 +24,22 @@ def extract_files_from_file(self, file_path, tmp_dir):
         self._initialize_shared_folder(tmp_dir)
         shutil.copy2(file_path, str(Path(tmp_dir, 'input', Path(file_path).name)))
 
-        result = run_docker_container(
-            'fkiecad/fact_extractor',
-            combine_stderr_stdout=True,
-            privileged=True,
-            mem_limit=f"{self.config.get('unpack', 'memory-limit', fallback='1024')}m",
-            mounts=[
-                Mount('/dev/', '/dev/', type='bind'),
-                Mount('/tmp/extractor', tmp_dir, type='bind'),
-            ],
-            command=f'--chown {getuid()}:{getgid()}'
-        )
+        try:
+            result = run_docker_container(
+                'fkiecad/fact_extractor',
+                combine_stderr_stdout=True,
+                privileged=True,
+                mem_limit=f"{self.config.get('unpack', 'memory-limit', fallback='1024')}m",
+                mounts=[
+                    Mount('/dev/', '/dev/', type='bind'),
+                    Mount('/tmp/extractor', tmp_dir, type='bind'),
+                ],
+                command=f'--chown {getuid()}:{getgid()}'
+            )
+        except exceptions.RequestException as err:
+            warning = f'Request exception executing docker extractor:\n{err}'
+            logging.warning(warning)
+            return None
 
         try:
             result.check_returncode()

src/web_interface/templates/analysis_plugins/unpacker.html

@@ -27,12 +27,12 @@
 	{% endif %}
 
 	{% for unpacker_meta_field in firmware.processed_analysis[selected_analysis].keys() %}
-		{% if unpacker_meta_field not in ['plugin_used', 'number_of_unpacked_files', 'output', 'analysis_date', 'plugin_version', 'summary'] %}
+		{% if unpacker_meta_field not in ['tags', 'plugin_used', 'number_of_unpacked_files', 'output', 'analysis_date', 'plugin_version', 'summary'] %}
 			<tr>
 				<td>{{ unpacker_meta_field | replace_underscore }}</td>
 				<td><pre><code style="word-break: break-all; white-space: pre-wrap;">{{ firmware.processed_analysis[selected_analysis][unpacker_meta_field] | nice_generic | safe }}</code></pre></td>
 			</tr>
 		{% endif %}			
 	{% endfor %}
 
-{% endblock %}
+{% endblock %}