Crane should continue to load rest of the metadata after corrupted da…

…ta is encountered

closes issue #3111
https://pulp.plan.io/issues/3111

crane/data.py

@@ -173,14 +173,18 @@ def load_all(app):
                  for f in fnmatch.filter(files, '*.json')]
         # load data from each file
         for metadata_file_path in paths:
-            repo_id, repo_tuple, image_ids = load_from_file(metadata_file_path)
+            try:
+                repo_id, repo_tuple, image_ids = load_from_file(metadata_file_path)
+            except Exception, e:
+                logger.error('skipping current metadata load: %s' % str(e))
+                continue
+
             if isinstance(repo_tuple, V1Repo):
                 v1_repos[repo_id] = repo_tuple
                 for image_id in image_ids:
                     images.setdefault(image_id, set()).add(repo_id)
             else:
                 v2_repos[repo_id] = repo_tuple
-
         # make each set immutable
         for image_id in images.keys():
             images[image_id] = frozenset(images[image_id])

tests/test_data.py

@@ -173,7 +173,22 @@ def test_with_metadata_bad(self, mock_error, mock_walk):
 
     @mock.patch.object(data.logger, 'error', spec_set=True)
     @mock.patch('os.walk', return_value=[
-                (demo_data.metadata_bad_path_v2, ('', ), ('wrong_version_2.json', ))])
+        (demo_data.metadata_bad_path, ('',), ('invalid_link.json1',))])
+    def test_with_metadata_bad_link(self, mock_error, mock_walk):
+        mock_app = mock.MagicMock()
+
+        data.load_all(mock_app)
+
+        # make sure the response data was not changed
+        self.assertEqual(data.v1_response_data['repos'], {})
+        self.assertEqual(data.v1_response_data['images'], {})
+
+        # make sure an error was logged
+        self.assertEqual(mock_error.call_count, 1)
+
+    @mock.patch.object(data.logger, 'error', spec_set=True)
+    @mock.patch('os.walk', return_value=[
+                (demo_data.metadata_bad_path_v2, ('', ), ('invalid_link_2.json', ))])
     def test_with_wrong_path(self, mock_error, mock_walk):
         mock_app = mock.MagicMock()
 
@@ -185,6 +200,20 @@ def test_with_wrong_path(self, mock_error, mock_walk):
         # make sure an error was logged
         self.assertEqual(mock_error.call_count, 1)
 
+    @mock.patch.object(data.logger, 'error', spec_set=True)
+    @mock.patch('os.walk', return_value=[
+        (demo_data.metadata_bad_path_v2, ('',), ('wrong_version_2.json1',))])
+    def test_with_wrong_path_v2_bad_link(self, mock_error, mock_walk):
+        mock_app = mock.MagicMock()
+
+        data.load_all(mock_app)
+
+        # make sure the response data was not changed
+        self.assertEqual(data.v2_response_data['repos'], {})
+
+        # make sure an error was logged
+        self.assertEqual(mock_error.call_count, 1)
+
 
 class StopTest(Exception):
     pass