[WIP] Feat: Maintain archetype-catalog.xml for Maven

charon/constants.py

@@ -13,6 +13,20 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+ARCHETYPE_CATALOG_FILENAME = "archetype-catalog.xml"
+ARCHETYPE_CATALOG_TEMPLATE = '''
+<archetype-catalog>
+  <archetypes>
+  {% for arch in archetypes %}
+    <archetype>
+      <groupId>{{ arch.group_id }}</groupId>
+      <artifactId>{{ arch.artifact_id }}</artifactId>
+      <version>{{ arch.version }}</version>
+      <description>{{ arch.description }}</description>
+    </archetype>{% endfor %}
+  </archetypes>
+</archetype-catalog>
+'''
 # Logging format used
 CHARON_LOGGING_FMT = '%(asctime)s - %(levelname)s - %(message)s'
 DESCRIPTION = "charon is a tool to synchronize several types of artifacts "

charon/storage.py

@@ -140,7 +140,7 @@ def path_upload_handler(full_file_path: str, path: str) -> bool:
                                 full_file_path,
                                 ExtraArgs={'ContentType': content_type}
                             )
-                    logger.info('Uploaded %s to bucket %s', full_file_path, bucket_name)
+                    logger.info('Uploaded %s to bucket %s', path, bucket_name)
                     uploaded_files.append(path_key)
                 except (ClientError, HTTPClientError) as e:
                     logger.error("ERROR: file %s not uploaded to bucket"
@@ -208,16 +208,18 @@ def path_upload_handler(full_file_path: str, path: str):
                 return False
             logger.info('Updating metadata %s to bucket %s', path, bucket_name)
             path_key = os.path.join(key_prefix, path) if key_prefix else path
-            fileObject = bucket.Object(path_key)
-            existed = self.file_exists(fileObject)
+            file_object = bucket.Object(path_key)
+            existed = self.file_exists(file_object)
             f_meta = {}
             need_overwritten = True
+
             sha1 = read_sha1(full_file_path)
+
             (content_type, _) = mimetypes.guess_type(full_file_path)
             if not content_type:
                 content_type = DEFAULT_MIME_TYPE
             if existed:
-                f_meta = fileObject.metadata
+                f_meta = file_object.metadata
                 need_overwritten = (
                     CHECKSUM_META_KEY not in f_meta or sha1 != f_meta[CHECKSUM_META_KEY]
                 )
@@ -234,15 +236,16 @@ def path_upload_handler(full_file_path: str, path: str):
             try:
                 if not self.dry_run:
                     if need_overwritten:
-                        fileObject.put(
+                        file_object.put(
                             Body=open(full_file_path, "rb"),
                             Metadata=f_meta,
                             ContentType=content_type
                         )
+
                     else:
                         # Should we update the s3 object metadata for metadata files?
                         try:
-                            self.__update_file_metadata(fileObject, bucket_name, path_key, f_meta)
+                            self.__update_file_metadata(file_object, bucket_name, path_key, f_meta)
                         except (ClientError, HTTPClientError) as e:
                             logger.error("ERROR: metadata %s not updated to bucket"
                                          " %s due to error: %s ", full_file_path,
@@ -263,7 +266,7 @@ def path_upload_handler(full_file_path: str, path: str):
 
     def delete_files(
         self, file_paths: List[str], bucket_name: str,
-        product: str, root="/", key_prefix: str = None
+        product: Optional[str], root="/", key_prefix: str = None
     ) -> Tuple[List[str], List[str]]:
         """ Deletes a list of files to s3 bucket. * Use the cut down file path as s3 key. The cut
         down way is move root from the file path if it starts with root. Example: if file_path is
@@ -285,13 +288,20 @@ def path_delete_handler(full_file_path: str, path: str):
             fileObject = bucket.Object(path_key)
             existed = self.file_exists(fileObject)
             if existed:
+                # NOTE: If we're NOT using the product key to track collisions
+                # (in the case of metadata), then this prods array will remain
+                # empty, and we will just delete the file, below. Otherwise,
+                # the product reference counts will be used (from object metadata).
                 prods = []
-                try:
-                    prods = fileObject.metadata[PRODUCT_META_KEY].split(",")
-                except KeyError:
-                    pass
-                if product and product in prods:
-                    prods.remove(product)
+                if product:
+                    try:
+                        prods = fileObject.metadata[PRODUCT_META_KEY].split(",")
+                    except KeyError:
+                        pass
+
+                    if product in prods:
+                        prods.remove(product)
+
                 if len(prods) > 0:
                     try:
                         logger.info(

requirements.txt

@@ -5,3 +5,4 @@ botocore
 click
 requests
 ruamel.yaml
+defusedxml

template/archetype-catalog.xml.j2

@@ -0,0 +1,11 @@
+<archetype-catalog>
+  <archetypes>
+  {% for arch in archetypes %}
+    <archetype>
+      <groupId>{{ arch.group_id }}</groupId>
+      <artifactId>{{ arch.artifact_id }}</artifactId>
+      <version>{{ arch.version }}</version>
+      <description>{{ arch.description }}</description>
+    </archetype>{% endfor %}
+  </archetypes>
+</archetype-catalog>

tests/commons.py

@@ -37,6 +37,14 @@
     "commons-logging/commons-logging/maven-metadata.xml.sha256"
 ]
 
+ARCHETYPE_CATALOG = "archetype-catalog.xml"
+ARCHETYPE_CATALOG_FILES = [
+    ARCHETYPE_CATALOG,
+    "archetype-catalog.xml.sha1",
+    "archetype-catalog.xml.md5",
+    "archetype-catalog.xml.sha256"
+]
+
 NON_MVN_FILES = [
     "commons-client-4.5.6/example-settings.xml",
     "commons-client-4.5.6/licenses/gnu",

tests/test_maven_del.py

@@ -4,7 +4,8 @@
 from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, BaseTest
 from tests.commons import (
     TEST_MVN_BUCKET, COMMONS_CLIENT_456_FILES, COMMONS_CLIENT_METAS,
-    COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS
+    COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS, ARCHETYPE_CATALOG,
+    ARCHETYPE_CATALOG_FILES
 )
 from moto import mock_s3
 import boto3
@@ -38,22 +39,24 @@ def test_maven_deletion(self):
         product_456 = "commons-client-4.5.6"
         handle_maven_del(
             test_zip, product_456,
-            bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir, do_index=False
+            bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir,
+            do_index=False
         )
 
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(18, len(actual_files))
+        self.assertEqual(22, len(actual_files))
 
         for f in COMMONS_CLIENT_456_FILES:
             self.assertNotIn(f, actual_files)
-        for f in COMMONS_CLIENT_METAS:
-            self.assertIn(f, actual_files)
 
-        for f in COMMONS_LOGGING_FILES:
-            self.assertIn(f, actual_files)
-        for f in COMMONS_LOGGING_METAS:
+        file_set = [
+            *COMMONS_CLIENT_METAS, *ARCHETYPE_CATALOG_FILES,
+            *COMMONS_LOGGING_FILES, *COMMONS_LOGGING_METAS
+        ]
+
+        for f in file_set:
             self.assertIn(f, actual_files)
 
         for obj in objs:
@@ -74,6 +77,14 @@ def test_maven_deletion(self):
         self.assertIn("<release>4.5.9</release>", meta_content_client)
         self.assertIn("<version>4.5.9</version>", meta_content_client)
 
+        meta_obj_cat = test_bucket.Object(ARCHETYPE_CATALOG)
+        meta_content_cat = str(meta_obj_cat.get()["Body"].read(), "utf-8")
+        self.assertIn(
+            "<groupId>org.apache.httpcomponents</groupId>", meta_content_cat
+        )
+        self.assertIn("<artifactId>httpclient</artifactId>", meta_content_cat)
+        self.assertNotIn("<version>4.5.6</version>", meta_content_cat)
+
         meta_obj_logging = test_bucket.Object(COMMONS_LOGGING_METAS[0])
         self.assertNotIn(PRODUCT_META_KEY, meta_obj_logging.metadata)
         meta_content_logging = str(meta_obj_logging.get()["Body"].read(), "utf-8")
@@ -112,7 +123,7 @@ def test_ignore_del(self):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(20, len(actual_files))
+        self.assertEqual(24, len(actual_files))
 
         httpclient_ignored_files = [
             "org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.pom.sha1",
@@ -173,7 +184,7 @@ def __test_prefix_deletion(self, prefix: str):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(18, len(actual_files))
+        self.assertEqual(22, len(actual_files))
 
         prefix_ = remove_prefix(prefix, "/")
         PREFIXED_COMMONS_CLIENT_456_FILES = [

tests/test_maven_index.py

@@ -60,7 +60,7 @@ def test_uploading_index(self):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(27, len(actual_files))
+        self.assertEqual(31, len(actual_files))
 
         for f in COMMONS_LOGGING_INDEXES:
             self.assertIn(f, actual_files)
@@ -113,7 +113,7 @@ def test_overlap_upload_index(self):
 
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
-        self.assertEqual(32, len(objs))
+        self.assertEqual(36, len(objs))
 
         indedx_obj = test_bucket.Object(COMMONS_CLIENT_INDEX)
         index_content = str(indedx_obj.get()["Body"].read(), "utf-8")
@@ -164,7 +164,7 @@ def __test_upload_index_with_prefix(self, prefix: str):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(27, len(actual_files))
+        self.assertEqual(31, len(actual_files))
 
         prefix_ = remove_prefix(prefix, "/")
         PREFIXED_LOGGING_INDEXES = [
@@ -221,7 +221,7 @@ def test_deletion_index(self):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(27, len(actual_files))
+        self.assertEqual(31, len(actual_files))
 
         for assert_file in COMMONS_CLIENT_459_INDEXES:
             self.assertIn(assert_file, actual_files)
@@ -288,7 +288,7 @@ def __test_deletion_index_with_prefix(self, prefix: str):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(27, len(actual_files))
+        self.assertEqual(31, len(actual_files))
 
         prefix_ = remove_prefix(prefix, "/")
         PREFIXED_459_INDEXES = [os.path.join(prefix_, i) for i in COMMONS_CLIENT_459_INDEXES]

tests/test_maven_upload.py

@@ -6,7 +6,7 @@
 from tests.commons import (
     TEST_MVN_BUCKET, COMMONS_CLIENT_456_FILES, COMMONS_CLIENT_459_FILES,
     COMMONS_CLIENT_METAS, COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS,
-    NON_MVN_FILES
+    NON_MVN_FILES, ARCHETYPE_CATALOG, ARCHETYPE_CATALOG_FILES
 )
 from moto import mock_s3
 import boto3
@@ -38,23 +38,24 @@ def test_fresh_upload(self):
         product = "commons-client-4.5.6"
         handle_maven_uploading(
             test_zip, product,
-            bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir, do_index=False
+            bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir,
+            do_index=False
         )
 
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(18, len(actual_files))
+        self.assertEqual(22, len(actual_files))
 
-        for f in COMMONS_CLIENT_456_FILES:
-            self.assertIn(f, actual_files)
-        for f in COMMONS_CLIENT_METAS:
-            self.assertIn(f, actual_files)
+        filesets = [
+            COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES,
+            COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS,
+            ARCHETYPE_CATALOG_FILES
+        ]
 
-        for f in COMMONS_LOGGING_FILES:
-            self.assertIn(f, actual_files)
-        for f in COMMONS_LOGGING_METAS:
-            self.assertIn(f, actual_files)
+        for fileset in filesets:
+            for f in fileset:
+                self.assertIn(f, actual_files)
 
         for f in NON_MVN_FILES:
             self.assertNotIn(f, actual_files)
@@ -92,6 +93,12 @@ def test_fresh_upload(self):
         self.assertIn("<latest>1.2</latest>", meta_content_logging)
         self.assertIn("<release>1.2</release>", meta_content_logging)
 
+        catalog = test_bucket.Object(ARCHETYPE_CATALOG)
+        cat_content = str(catalog.get()["Body"].read(), "utf-8")
+        self.assertIn("<version>4.5.6</version>", cat_content)
+        self.assertIn("<artifactId>httpclient</artifactId>", cat_content)
+        self.assertIn("<groupId>org.apache.httpcomponents</groupId>", cat_content)
+
     def test_overlap_upload(self):
         test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip")
         product_456 = "commons-client-4.5.6"
@@ -110,20 +117,17 @@ def test_overlap_upload(self):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(22, len(actual_files))
+        self.assertEqual(26, len(actual_files))
+
+        filesets = [
+            COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES,
+            COMMONS_CLIENT_459_FILES,
+            ARCHETYPE_CATALOG_FILES
+        ]
+        for fileset in filesets:
+            for f in fileset:
+                self.assertIn(f, actual_files)
 
-        for f in COMMONS_CLIENT_456_FILES:
-            self.assertIn(f, actual_files)
-            self.assertEqual(
-                product_456, test_bucket.Object(f).metadata[PRODUCT_META_KEY]
-            )
-        for f in COMMONS_CLIENT_459_FILES:
-            self.assertIn(f, actual_files)
-            self.assertEqual(
-                product_459, test_bucket.Object(f).metadata[PRODUCT_META_KEY]
-            )
-        for f in COMMONS_CLIENT_METAS:
-            self.assertIn(f, actual_files)
         product_mix = set([product_456, product_459])
 
         for f in COMMONS_LOGGING_FILES:
@@ -154,6 +158,13 @@ def test_overlap_upload(self):
         self.assertIn("<latest>1.2</latest>", meta_content_logging)
         self.assertIn("<release>1.2</release>", meta_content_logging)
 
+        catalog = test_bucket.Object(ARCHETYPE_CATALOG)
+        cat_content = str(catalog.get()["Body"].read(), "utf-8")
+        self.assertIn("<version>4.5.6</version>", cat_content)
+        self.assertIn("<version>4.5.9</version>", cat_content)
+        self.assertIn("<artifactId>httpclient</artifactId>", cat_content)
+        self.assertIn("<groupId>org.apache.httpcomponents</groupId>", cat_content)
+
     def test_ignore_upload(self):
         test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip")
         product_456 = "commons-client-4.5.6"
@@ -165,7 +176,7 @@ def test_ignore_upload(self):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(13, len(actual_files))
+        self.assertEqual(17, len(actual_files))
 
         ignored_files = [
             "org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.pom.sha1",
@@ -201,7 +212,7 @@ def __test_prefix_upload(self, prefix: str):
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
         actual_files = [obj.key for obj in objs]
-        self.assertEqual(18, len(actual_files))
+        self.assertEqual(22, len(actual_files))
 
         prefix_ = remove_prefix(prefix, "/")
         PREFIXED_COMMONS_CLIENT_456_FILES = [

tests/test_pkgs_dryrun.py

@@ -55,7 +55,7 @@ def test_maven_delete_dry_run(self):
 
         test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
         objs = list(test_bucket.objects.all())
-        self.assertEqual(32, len(objs))
+        self.assertEqual(36, len(objs))
 
     def test_npm_upload_dry_run(self):
         test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz")