diff --git a/charon/constants.py b/charon/constants.py
index 80d2c8f8..18586eb5 100644
--- a/charon/constants.py
+++ b/charon/constants.py
@@ -13,6 +13,20 @@
See the License for the specific language governing permissions and
limitations under the License.
"""
+ARCHETYPE_CATALOG_FILENAME = "archetype-catalog.xml"
+ARCHETYPE_CATALOG_TEMPLATE = '''
+
+
+ {% for arch in archetypes %}
+
+ {{ arch.group_id }}
+ {{ arch.artifact_id }}
+ {{ arch.version }}
+ {{ arch.description }}
+ {% endfor %}
+
+
+'''
# Logging format used
CHARON_LOGGING_FMT = '%(asctime)s - %(levelname)s - %(message)s'
DESCRIPTION = "charon is a tool to synchronize several types of artifacts "
diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py
index 6cf9712b..3790132f 100644
--- a/charon/pkgs/maven.py
+++ b/charon/pkgs/maven.py
@@ -22,12 +22,15 @@
from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process
from charon.config import get_template
from charon.constants import (META_FILE_GEN_KEY, META_FILE_DEL_KEY,
- META_FILE_FAILED, MAVEN_METADATA_TEMPLATE)
+ META_FILE_FAILED, MAVEN_METADATA_TEMPLATE,
+ ARCHETYPE_CATALOG_TEMPLATE, ARCHETYPE_CATALOG_FILENAME)
from typing import Dict, List, Tuple
from jinja2 import Template
from datetime import datetime
from zipfile import ZipFile, BadZipFile
from tempfile import mkdtemp
+from defusedxml import ElementTree
+
import os
import sys
import logging
@@ -37,17 +40,19 @@
logger = logging.getLogger(__name__)
-def __get_mvn_template() -> str:
- """Gets the jinja2 template file content for maven-metadata.xml generation"""
+def __get_mvn_template(kind: str, default: str) -> str:
+ """Gets the jinja2 template file content for metadata generation"""
try:
- return get_template("maven-metadata.xml.j2")
+ return get_template(kind)
except FileNotFoundError:
- logger.info("maven-metadata.xml template file not defined,"
- " will use default template.")
- return MAVEN_METADATA_TEMPLATE
+ logger.info("%s template file not defined,"
+ " will use default template.", kind)
+ return default
-META_TEMPLATE = __get_mvn_template()
+META_TEMPLATE = __get_mvn_template("maven-metadata.xml.j2", MAVEN_METADATA_TEMPLATE)
+ARCH_TEMPLATE = __get_mvn_template("archetype-catalog.xml.j2", ARCHETYPE_CATALOG_TEMPLATE)
+STANDARD_GENERATED_IGNORES = ["maven-metadata.xml", "archetype-catalog.xml"]
class MavenMetadata(object):
@@ -85,6 +90,48 @@ def __str__(self) -> str:
return f"{self.group_id}:{self.artifact_id}\n{self.versions}\n\n"
+class ArchetypeRef(object):
+ """This ArchetypeRef will represent an entry in archetype-catalog.xml content which will be
+ used in jinja2 or other places
+ """
+
+ def __init__(self, group_id: str, artifact_id: str, version: str, description: str):
+ self.group_id = group_id
+ self.artifact_id = artifact_id
+ self.version = version
+ self.description = description
+
+ def __hash__(self):
+ return hash(self.group_id + self.artifact_id + self.version)
+
+ def __eq__(self, other) -> bool:
+ if isinstance(other, ArchetypeRef):
+ return self.group_id == other.group_id \
+ and self.artifact_id == other.artifact_id \
+ and self.version == other.version
+
+ return False
+
+ def __str__(self) -> str:
+ return f"{self.group_id}:{self.artifact_id}\n{self.version}\n{self.description}\n\n"
+
+
+class MavenArchetypeCatalog(object):
+ """This MavenArchetypeCatalog represents an archetype-catalog.xml which will be
+ used in jinja2 to regenerate the file with merged contents
+ """
+
+ def __init__(self, archetypes: List[ArchetypeRef]):
+ self.archetypes = sorted(set(archetypes), key=ArchetypeCompareKey)
+
+ def generate_meta_file_content(self) -> str:
+ template = Template(ARCHETYPE_CATALOG_TEMPLATE)
+ return template.render(archetypes=self.archetypes)
+
+ def __str__(self) -> str:
+ return f"(Archetype Catalog with {len(self.archetypes)} entries).\n\n"
+
+
def scan_for_poms(full_path: str) -> List[str]:
"""Scan a file path and finds all pom files absolute paths"""
# collect poms
@@ -173,18 +220,24 @@ def gen_meta_file(group_id, artifact_id: str, versions: list, root="/", digest=T
except FileNotFoundError as e:
raise e
if digest:
- md5_path = final_meta_path + ".md5"
- sha1_path = final_meta_path + ".sha1"
- sha256_path = final_meta_path + ".sha256"
- if __gen_digest_file(md5_path, final_meta_path, HashType.MD5):
- meta_files.append(md5_path)
- if __gen_digest_file(sha1_path, final_meta_path, HashType.SHA1):
- meta_files.append(sha1_path)
- if __gen_digest_file(sha256_path, final_meta_path, HashType.SHA256):
- meta_files.append(sha256_path)
+ meta_files.extend(__gen_all_digest_files(final_meta_path))
return meta_files
+def __gen_all_digest_files(meta_file_path: str) -> List[str]:
+ md5_path = meta_file_path + ".md5"
+ sha1_path = meta_file_path + ".sha1"
+ sha256_path = meta_file_path + ".sha256"
+ digest_files = []
+ if __gen_digest_file(md5_path, meta_file_path, HashType.MD5):
+ digest_files.append(md5_path)
+ if __gen_digest_file(sha1_path, meta_file_path, HashType.SHA1):
+ digest_files.append(sha1_path)
+ if __gen_digest_file(sha256_path, meta_file_path, HashType.SHA256):
+ digest_files.append(sha256_path)
+ return digest_files
+
+
def __gen_digest_file(hash_file_path, meta_file_path: str, hashtype: HashType) -> bool:
try:
overwrite_file(hash_file_path, digest(meta_file_path, hashtype))
@@ -280,6 +333,31 @@ def handle_maven_uploading(
failed_metas.extend(_failed_metas)
logger.info("maven-metadata.xml updating done\n")
+ # 7. Determine refreshment of archetype-catalog.xml
+ if os.path.exists(os.path.join(top_level, "archetype-catalog.xml")):
+ logger.info("Start generating archetype-catalog.xml")
+ upload_archetype_file = _generate_upload_archetype_catalog(
+ s3=s3_client, bucket=bucket,
+ root=top_level,
+ prefix=prefix_
+ )
+ logger.info("archetype-catalog.xml files generation done\n")
+
+ # 8. Upload archetype-catalog.xml if it has changed
+ if upload_archetype_file:
+ archetype_files = [os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)]
+ archetype_files.extend(__hash_decorate_metadata(top_level, ARCHETYPE_CATALOG_FILENAME))
+ logger.info("Start updating archetype-catalog.xml to s3")
+ (_, _failed_metas) = s3_client.upload_metadatas(
+ meta_file_paths=archetype_files,
+ bucket_name=bucket,
+ product=None,
+ root=top_level,
+ key_prefix=prefix_
+ )
+ failed_metas.extend(_failed_metas)
+ logger.info("archetype-catalog.xml updating done\n")
+
# this step generates index.html for each dir and add them to file list
# index is similar to metadata, it will be overwritten everytime
if do_index:
@@ -399,6 +477,40 @@ def handle_maven_del(
failed_metas.extend(_failed_metas)
logger.info("maven-metadata.xml updating done\n")
+ # 7. Determine refreshment of archetype-catalog.xml
+ if os.path.exists(os.path.join(top_level, "archetype-catalog.xml")):
+ logger.info("Start generating archetype-catalog.xml")
+ archetype_action = _generate_rollback_archetype_catalog(
+ s3=s3_client, bucket=bucket,
+ root=top_level,
+ prefix=prefix_
+ )
+ logger.info("archetype-catalog.xml files generation done\n")
+
+ # 8. Upload or Delete archetype-catalog.xml if it has changed
+ archetype_files = [os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)]
+ archetype_files.extend(__hash_decorate_metadata(top_level, ARCHETYPE_CATALOG_FILENAME))
+ if archetype_action < 0:
+ logger.info("Start updating archetype-catalog.xml to s3")
+ (_, _failed_metas) = s3_client.delete_files(
+ file_paths=archetype_files,
+ bucket_name=bucket,
+ product=None,
+ root=top_level,
+ key_prefix=prefix_
+ )
+ failed_metas.extend(_failed_metas)
+ elif archetype_action > 0:
+ (_, _failed_metas) = s3_client.upload_metadatas(
+ meta_file_paths=archetype_files,
+ bucket_name=bucket,
+ product=None,
+ root=top_level,
+ key_prefix=prefix_
+ )
+ failed_metas.extend(_failed_metas)
+ logger.info("archetype-catalog.xml updating done\n")
+
if do_index:
logger.info("Start generating index files for all changed entries")
created_indexes = indexing.generate_indexes(
@@ -459,16 +571,20 @@ def _scan_paths(files_root: str, ignore_patterns: List[str],
for name in names:
path = os.path.join(root_dir, name)
- if _is_ignored(name, ignore_patterns):
- ignored_paths.append(name)
- continue
if top_level in root_dir:
+ # Let's wait to do the regex / pom examination until we
+ # know we're inside a valid root directory.
+ if _is_ignored(name, ignore_patterns):
+ ignored_paths.append(name)
+ continue
+
valid_mvn_paths.append(path)
+
+ if name.strip().endswith(".pom"):
+ logger.debug("Found pom %s", name)
+ valid_poms.append(path)
else:
non_mvn_paths.append(path)
- if name.strip().endswith(".pom"):
- logger.debug("Found pom %s", name)
- valid_poms.append(path)
if len(non_mvn_paths) > 0:
non_mvn_items = [n.replace(files_root, "") for n in non_mvn_paths]
@@ -497,6 +613,199 @@ def _scan_paths(files_root: str, ignore_patterns: List[str],
return (top_level, valid_mvn_paths, valid_poms, valid_dirs)
+def _generate_rollback_archetype_catalog(
+ s3: S3Client, bucket: str,
+ root: str, prefix: str = None
+) -> int:
+ """Determine whether the local archive contains /archetype-catalog.xml
+ in the repo contents.
+ If so, determine whether the archetype-catalog.xml is already
+ available in the bucket. Merge (or unmerge) these catalogs and
+ return an integer, indicating whether the bucket file should be
+ replaced (+1), deleted (-1), or, in the case where no action is
+ required, it will return NO-OP (0).
+
+ NOTE: There are three return values:
+ - +1 - UPLOAD the local catalog with its rolled back changes
+ - -1 - DELETE the (now empty) bucket catalog
+ - 0 - take no action
+ """
+ local = os.path.join(root, ARCHETYPE_CATALOG_FILENAME)
+ if prefix:
+ remote = os.path.join(prefix, ARCHETYPE_CATALOG_FILENAME)
+ else:
+ remote = ARCHETYPE_CATALOG_FILENAME
+
+ # If there is no local catalog, this is a NO-OP
+ if os.path.exists(local):
+ if not s3.file_exists_in_bucket(bucket, remote):
+ # If there is no catalog in the bucket...this is a NO-OP
+ return 0
+ else:
+ # If there IS a catalog in the bucket, we need to merge or un-merge it.
+ with open(local, "rb") as f:
+ local_archetypes = _parse_archetypes(f.read())
+
+ if len(local_archetypes) < 1:
+ # If there are no local archetypes in the catalog,
+ # there's nothing to do.
+ logger.warning(
+ "No archetypes found in local archetype-catalog.xml, "
+ "even though the file exists! Skipping."
+ )
+ return 0
+
+ else:
+ # Read the archetypes from the bucket so we can do a merge / un-merge
+ remote_xml = s3.read_file_content(bucket, remote)
+ remote_archetypes = _parse_archetypes(remote_xml)
+
+ if len(remote_archetypes) < 1:
+ # Nothing in the bucket. Clear out this empty file.
+ __gen_all_digest_files(local)
+ return -1
+
+ else:
+ # If we're deleting, un-merge the local archetypes from
+ # the remote ones.
+ #
+ # NOTE: The ONLY reason we can get away with this kind of
+ # naive un-merge is that products only bother to publish
+ # archetypes for their own direct users. If they publish
+ # an archetype, it's only for use with their product.
+ # Therefore, if we rollback that product, the archetypes
+ # they reference shouldn't be useful anymore.
+ for la in local_archetypes:
+ if la in remote_archetypes:
+ remote_archetypes.remove(la)
+
+ if len(remote_archetypes) < 1:
+ # If there are no remote archetypes left after removing
+ # ours DELETE the bucket catalog.
+ __gen_all_digest_files(local)
+ return -1
+ else:
+ # Re-render the result of our archetype un-merge to the
+ # local file, in preparation for upload.
+ with open(local, 'wb') as f:
+ content = MavenArchetypeCatalog(remote_archetypes)\
+ .generate_meta_file_content()
+ try:
+ overwrite_file(local, content)
+ except FileNotFoundError as e:
+ logger.error(
+ "Error: Can not create file %s because of some missing folders",
+ local,
+ )
+ raise e
+ __gen_all_digest_files(local)
+ return 1
+
+ return 0
+
+
+def _generate_upload_archetype_catalog(
+ s3: S3Client, bucket: str,
+ root: str, prefix: str = None
+) -> bool:
+ """Determine whether the local archive contains /archetype-catalog.xml
+ in the repo contents.
+ If so, determine whether the archetype-catalog.xml is already
+ available in the bucket. Merge (or unmerge) these catalogs and
+ return a boolean indicating whether the local file should be uploaded.
+ """
+ local = os.path.join(root, ARCHETYPE_CATALOG_FILENAME)
+ if prefix:
+ remote = os.path.join(prefix, ARCHETYPE_CATALOG_FILENAME)
+ else:
+ remote = ARCHETYPE_CATALOG_FILENAME
+
+ # If there is no local catalog, this is a NO-OP
+ if os.path.exists(local):
+ if not s3.file_exists_in_bucket(bucket, remote):
+ __gen_all_digest_files(local)
+ # If there is no catalog in the bucket, just upload what we have locally
+ return True
+ else:
+ # If there IS a catalog in the bucket, we need to merge or un-merge it.
+ with open(local, "rb") as f:
+ local_archetypes = _parse_archetypes(f.read())
+
+ if len(local_archetypes) < 1:
+ logger.warning(
+ "No archetypes found in local archetype-catalog.xml, "
+ "even though the file exists! Skipping."
+ )
+
+ else:
+ # Read the archetypes from the bucket so we can do a merge / un-merge
+ remote_xml = s3.read_file_content(bucket, remote)
+ remote_archetypes = _parse_archetypes(remote_xml)
+
+ if len(remote_archetypes) < 1:
+ __gen_all_digest_files(local)
+ # Nothing in the bucket. Just push what we have locally.
+ return True
+ else:
+ original_remote_size = len(remote_archetypes)
+ for la in local_archetypes:
+ # The cautious approach in this operation contradicts
+ # assumptions we make for the rollback case.
+ # That's because we should NEVER encounter a collision
+ # on archetype GAV...they should belong with specific
+ # product releases.
+ #
+ # Still, we will WARN, not ERROR if we encounter this.
+ if la not in remote_archetypes:
+ remote_archetypes.append(la)
+ else:
+ logger.warning(
+ "\n\n\nDUPLICATE ARCHETYPE: %s. "
+ "This makes rollback of the current release UNSAFE!\n\n\n",
+ la
+ )
+
+ if len(remote_archetypes) != original_remote_size:
+ # If the number of archetypes in the version of
+ # the file from the bucket has changed, we need
+ # to regenerate the file and re-upload it.
+ #
+ # Re-render the result of our archetype merge /
+ # un-merge to the local file, in preparation for
+ # upload.
+ with open(local, 'wb') as f:
+ content = MavenArchetypeCatalog(remote_archetypes)\
+ .generate_meta_file_content()
+ try:
+ overwrite_file(local, content)
+ except FileNotFoundError as e:
+ logger.error(
+ "Error: Can not create file %s because of some missing folders",
+ local,
+ )
+ raise e
+ __gen_all_digest_files(local)
+ return True
+
+ return False
+
+
+def _parse_archetypes(source) -> List[ArchetypeRef]:
+ tree = ElementTree.fromstring(
+ source.strip(), forbid_dtd=True,
+ forbid_entities=True, forbid_external=True
+ )
+ archetypes = []
+ for a in tree.findall("./archetypes/archetype"):
+ gid = a.find('groupId').text
+ aid = a.find('artifactId').text
+ ver = a.find('version').text
+ desc = a.find('description').text
+ archetypes.append(ArchetypeRef(gid, aid, ver, desc))
+
+ return archetypes
+
+
def _generate_metadatas(
s3: S3Client, bucket: str,
poms: List[str], root: str,
@@ -580,9 +889,11 @@ def __hash_decorate_metadata(path: str, metadata: str) -> List[str]:
def _is_ignored(filename: str, ignore_patterns: List[str]) -> bool:
- if "maven-metadata.xml" in filename:
- logger.warning("Ignoring Maven metadata file from input: %s", filename)
- return True
+ for ignored_name in STANDARD_GENERATED_IGNORES:
+ if ignored_name in filename:
+ logger.warning("Ignoring standard generated Maven path: %s", filename)
+ return True
+
if ignore_patterns:
for dirs in ignore_patterns:
if re.search(dirs, filename):
@@ -659,3 +970,22 @@ def __compare(self, other) -> int:
else:
continue
return 0
+
+
+class ArchetypeCompareKey(VersionCompareKey):
+ 'Used as key function for GAV sorting'
+ def __init__(self, gav):
+ super().__init__(gav.version)
+ self.gav = gav
+
+ # pylint: disable=unused-private-member
+ def __compare(self, other) -> int:
+ x = self.gav.group_id + ":" + self.gav.artifact_id
+ y = other.gav.group_id + ":" + other.gav.artifact_id
+
+ if x == y:
+ return 0
+ elif x < y:
+ return -1
+ else:
+ return 1
diff --git a/charon/storage.py b/charon/storage.py
index 976109f7..51d02dc9 100644
--- a/charon/storage.py
+++ b/charon/storage.py
@@ -140,7 +140,7 @@ def path_upload_handler(full_file_path: str, path: str) -> bool:
full_file_path,
ExtraArgs={'ContentType': content_type}
)
- logger.info('Uploaded %s to bucket %s', full_file_path, bucket_name)
+ logger.info('Uploaded %s to bucket %s', path, bucket_name)
uploaded_files.append(path_key)
except (ClientError, HTTPClientError) as e:
logger.error("ERROR: file %s not uploaded to bucket"
@@ -208,16 +208,18 @@ def path_upload_handler(full_file_path: str, path: str):
return False
logger.info('Updating metadata %s to bucket %s', path, bucket_name)
path_key = os.path.join(key_prefix, path) if key_prefix else path
- fileObject = bucket.Object(path_key)
- existed = self.file_exists(fileObject)
+ file_object = bucket.Object(path_key)
+ existed = self.file_exists(file_object)
f_meta = {}
need_overwritten = True
+
sha1 = read_sha1(full_file_path)
+
(content_type, _) = mimetypes.guess_type(full_file_path)
if not content_type:
content_type = DEFAULT_MIME_TYPE
if existed:
- f_meta = fileObject.metadata
+ f_meta = file_object.metadata
need_overwritten = (
CHECKSUM_META_KEY not in f_meta or sha1 != f_meta[CHECKSUM_META_KEY]
)
@@ -234,15 +236,16 @@ def path_upload_handler(full_file_path: str, path: str):
try:
if not self.dry_run:
if need_overwritten:
- fileObject.put(
+ file_object.put(
Body=open(full_file_path, "rb"),
Metadata=f_meta,
ContentType=content_type
)
+
else:
# Should we update the s3 object metadata for metadata files?
try:
- self.__update_file_metadata(fileObject, bucket_name, path_key, f_meta)
+ self.__update_file_metadata(file_object, bucket_name, path_key, f_meta)
except (ClientError, HTTPClientError) as e:
logger.error("ERROR: metadata %s not updated to bucket"
" %s due to error: %s ", full_file_path,
@@ -263,7 +266,7 @@ def path_upload_handler(full_file_path: str, path: str):
def delete_files(
self, file_paths: List[str], bucket_name: str,
- product: str, root="/", key_prefix: str = None
+ product: Optional[str], root="/", key_prefix: str = None
) -> Tuple[List[str], List[str]]:
""" Deletes a list of files to s3 bucket. * Use the cut down file path as s3 key. The cut
down way is move root from the file path if it starts with root. Example: if file_path is
@@ -285,13 +288,20 @@ def path_delete_handler(full_file_path: str, path: str):
fileObject = bucket.Object(path_key)
existed = self.file_exists(fileObject)
if existed:
+ # NOTE: If we're NOT using the product key to track collisions
+ # (in the case of metadata), then this prods array will remain
+ # empty, and we will just delete the file, below. Otherwise,
+ # the product reference counts will be used (from object metadata).
prods = []
- try:
- prods = fileObject.metadata[PRODUCT_META_KEY].split(",")
- except KeyError:
- pass
- if product and product in prods:
- prods.remove(product)
+ if product:
+ try:
+ prods = fileObject.metadata[PRODUCT_META_KEY].split(",")
+ except KeyError:
+ pass
+
+ if product in prods:
+ prods.remove(product)
+
if len(prods) > 0:
try:
logger.info(
diff --git a/requirements.txt b/requirements.txt
index 8f0aa0a7..14d787ed 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,4 @@ botocore
click
requests
ruamel.yaml
+defusedxml
diff --git a/template/archetype-catalog.xml.j2 b/template/archetype-catalog.xml.j2
new file mode 100644
index 00000000..fb6e3f64
--- /dev/null
+++ b/template/archetype-catalog.xml.j2
@@ -0,0 +1,11 @@
+
+
+ {% for arch in archetypes %}
+
+ {{ arch.group_id }}
+ {{ arch.artifact_id }}
+ {{ arch.version }}
+ {{ arch.description }}
+ {% endfor %}
+
+
diff --git a/tests/commons.py b/tests/commons.py
index c5dd93c3..61a61c4a 100644
--- a/tests/commons.py
+++ b/tests/commons.py
@@ -37,6 +37,14 @@
"commons-logging/commons-logging/maven-metadata.xml.sha256"
]
+ARCHETYPE_CATALOG = "archetype-catalog.xml"
+ARCHETYPE_CATALOG_FILES = [
+ ARCHETYPE_CATALOG,
+ "archetype-catalog.xml.sha1",
+ "archetype-catalog.xml.md5",
+ "archetype-catalog.xml.sha256"
+]
+
NON_MVN_FILES = [
"commons-client-4.5.6/example-settings.xml",
"commons-client-4.5.6/licenses/gnu",
diff --git a/tests/input/commons-client-4.5.6.zip b/tests/input/commons-client-4.5.6.zip
index 5522517e..7ce27974 100644
Binary files a/tests/input/commons-client-4.5.6.zip and b/tests/input/commons-client-4.5.6.zip differ
diff --git a/tests/input/commons-client-4.5.9.zip b/tests/input/commons-client-4.5.9.zip
index 02f3fa75..a42de5a5 100644
Binary files a/tests/input/commons-client-4.5.9.zip and b/tests/input/commons-client-4.5.9.zip differ
diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py
index d8637604..2aeeb27e 100644
--- a/tests/test_maven_del.py
+++ b/tests/test_maven_del.py
@@ -4,7 +4,8 @@
from tests.base import LONG_TEST_PREFIX, SHORT_TEST_PREFIX, BaseTest
from tests.commons import (
TEST_MVN_BUCKET, COMMONS_CLIENT_456_FILES, COMMONS_CLIENT_METAS,
- COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS
+ COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS, ARCHETYPE_CATALOG,
+ ARCHETYPE_CATALOG_FILES
)
from moto import mock_s3
import boto3
@@ -38,22 +39,24 @@ def test_maven_deletion(self):
product_456 = "commons-client-4.5.6"
handle_maven_del(
test_zip, product_456,
- bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir, do_index=False
+ bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir,
+ do_index=False
)
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(18, len(actual_files))
+ self.assertEqual(22, len(actual_files))
for f in COMMONS_CLIENT_456_FILES:
self.assertNotIn(f, actual_files)
- for f in COMMONS_CLIENT_METAS:
- self.assertIn(f, actual_files)
- for f in COMMONS_LOGGING_FILES:
- self.assertIn(f, actual_files)
- for f in COMMONS_LOGGING_METAS:
+ file_set = [
+ *COMMONS_CLIENT_METAS, *ARCHETYPE_CATALOG_FILES,
+ *COMMONS_LOGGING_FILES, *COMMONS_LOGGING_METAS
+ ]
+
+ for f in file_set:
self.assertIn(f, actual_files)
for obj in objs:
@@ -74,6 +77,14 @@ def test_maven_deletion(self):
self.assertIn("4.5.9", meta_content_client)
self.assertIn("4.5.9", meta_content_client)
+ meta_obj_cat = test_bucket.Object(ARCHETYPE_CATALOG)
+ meta_content_cat = str(meta_obj_cat.get()["Body"].read(), "utf-8")
+ self.assertIn(
+ "org.apache.httpcomponents", meta_content_cat
+ )
+ self.assertIn("httpclient", meta_content_cat)
+ self.assertNotIn("4.5.6", meta_content_cat)
+
meta_obj_logging = test_bucket.Object(COMMONS_LOGGING_METAS[0])
self.assertNotIn(PRODUCT_META_KEY, meta_obj_logging.metadata)
meta_content_logging = str(meta_obj_logging.get()["Body"].read(), "utf-8")
@@ -112,7 +123,7 @@ def test_ignore_del(self):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(20, len(actual_files))
+ self.assertEqual(24, len(actual_files))
httpclient_ignored_files = [
"org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.pom.sha1",
@@ -173,7 +184,7 @@ def __test_prefix_deletion(self, prefix: str):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(18, len(actual_files))
+ self.assertEqual(22, len(actual_files))
prefix_ = remove_prefix(prefix, "/")
PREFIXED_COMMONS_CLIENT_456_FILES = [
diff --git a/tests/test_maven_index.py b/tests/test_maven_index.py
index 2ed718b7..9df90efd 100644
--- a/tests/test_maven_index.py
+++ b/tests/test_maven_index.py
@@ -60,7 +60,7 @@ def test_uploading_index(self):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(27, len(actual_files))
+ self.assertEqual(31, len(actual_files))
for f in COMMONS_LOGGING_INDEXES:
self.assertIn(f, actual_files)
@@ -113,7 +113,7 @@ def test_overlap_upload_index(self):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
- self.assertEqual(32, len(objs))
+ self.assertEqual(36, len(objs))
indedx_obj = test_bucket.Object(COMMONS_CLIENT_INDEX)
index_content = str(indedx_obj.get()["Body"].read(), "utf-8")
@@ -164,7 +164,7 @@ def __test_upload_index_with_prefix(self, prefix: str):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(27, len(actual_files))
+ self.assertEqual(31, len(actual_files))
prefix_ = remove_prefix(prefix, "/")
PREFIXED_LOGGING_INDEXES = [
@@ -221,7 +221,7 @@ def test_deletion_index(self):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(27, len(actual_files))
+ self.assertEqual(31, len(actual_files))
for assert_file in COMMONS_CLIENT_459_INDEXES:
self.assertIn(assert_file, actual_files)
@@ -288,7 +288,7 @@ def __test_deletion_index_with_prefix(self, prefix: str):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(27, len(actual_files))
+ self.assertEqual(31, len(actual_files))
prefix_ = remove_prefix(prefix, "/")
PREFIXED_459_INDEXES = [os.path.join(prefix_, i) for i in COMMONS_CLIENT_459_INDEXES]
diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py
index 46b73cc9..a2549a8f 100644
--- a/tests/test_maven_upload.py
+++ b/tests/test_maven_upload.py
@@ -6,7 +6,7 @@
from tests.commons import (
TEST_MVN_BUCKET, COMMONS_CLIENT_456_FILES, COMMONS_CLIENT_459_FILES,
COMMONS_CLIENT_METAS, COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS,
- NON_MVN_FILES
+ NON_MVN_FILES, ARCHETYPE_CATALOG, ARCHETYPE_CATALOG_FILES
)
from moto import mock_s3
import boto3
@@ -38,23 +38,24 @@ def test_fresh_upload(self):
product = "commons-client-4.5.6"
handle_maven_uploading(
test_zip, product,
- bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir, do_index=False
+ bucket_name=TEST_MVN_BUCKET, dir_=self.tempdir,
+ do_index=False
)
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(18, len(actual_files))
+ self.assertEqual(22, len(actual_files))
- for f in COMMONS_CLIENT_456_FILES:
- self.assertIn(f, actual_files)
- for f in COMMONS_CLIENT_METAS:
- self.assertIn(f, actual_files)
+ filesets = [
+ COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES,
+ COMMONS_LOGGING_FILES, COMMONS_LOGGING_METAS,
+ ARCHETYPE_CATALOG_FILES
+ ]
- for f in COMMONS_LOGGING_FILES:
- self.assertIn(f, actual_files)
- for f in COMMONS_LOGGING_METAS:
- self.assertIn(f, actual_files)
+ for fileset in filesets:
+ for f in fileset:
+ self.assertIn(f, actual_files)
for f in NON_MVN_FILES:
self.assertNotIn(f, actual_files)
@@ -92,6 +93,12 @@ def test_fresh_upload(self):
self.assertIn("1.2", meta_content_logging)
self.assertIn("1.2", meta_content_logging)
+ catalog = test_bucket.Object(ARCHETYPE_CATALOG)
+ cat_content = str(catalog.get()["Body"].read(), "utf-8")
+ self.assertIn("4.5.6", cat_content)
+ self.assertIn("httpclient", cat_content)
+ self.assertIn("org.apache.httpcomponents", cat_content)
+
def test_overlap_upload(self):
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip")
product_456 = "commons-client-4.5.6"
@@ -110,20 +117,17 @@ def test_overlap_upload(self):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(22, len(actual_files))
+ self.assertEqual(26, len(actual_files))
+
+ filesets = [
+ COMMONS_CLIENT_METAS, COMMONS_CLIENT_456_FILES,
+ COMMONS_CLIENT_459_FILES,
+ ARCHETYPE_CATALOG_FILES
+ ]
+ for fileset in filesets:
+ for f in fileset:
+ self.assertIn(f, actual_files)
- for f in COMMONS_CLIENT_456_FILES:
- self.assertIn(f, actual_files)
- self.assertEqual(
- product_456, test_bucket.Object(f).metadata[PRODUCT_META_KEY]
- )
- for f in COMMONS_CLIENT_459_FILES:
- self.assertIn(f, actual_files)
- self.assertEqual(
- product_459, test_bucket.Object(f).metadata[PRODUCT_META_KEY]
- )
- for f in COMMONS_CLIENT_METAS:
- self.assertIn(f, actual_files)
product_mix = set([product_456, product_459])
for f in COMMONS_LOGGING_FILES:
@@ -154,6 +158,13 @@ def test_overlap_upload(self):
self.assertIn("1.2", meta_content_logging)
self.assertIn("1.2", meta_content_logging)
+ catalog = test_bucket.Object(ARCHETYPE_CATALOG)
+ cat_content = str(catalog.get()["Body"].read(), "utf-8")
+ self.assertIn("4.5.6", cat_content)
+ self.assertIn("4.5.9", cat_content)
+ self.assertIn("httpclient", cat_content)
+ self.assertIn("org.apache.httpcomponents", cat_content)
+
def test_ignore_upload(self):
test_zip = os.path.join(os.getcwd(), "tests/input/commons-client-4.5.6.zip")
product_456 = "commons-client-4.5.6"
@@ -165,7 +176,7 @@ def test_ignore_upload(self):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(13, len(actual_files))
+ self.assertEqual(17, len(actual_files))
ignored_files = [
"org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.pom.sha1",
@@ -201,7 +212,7 @@ def __test_prefix_upload(self, prefix: str):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
actual_files = [obj.key for obj in objs]
- self.assertEqual(18, len(actual_files))
+ self.assertEqual(22, len(actual_files))
prefix_ = remove_prefix(prefix, "/")
PREFIXED_COMMONS_CLIENT_456_FILES = [
diff --git a/tests/test_pkgs_dryrun.py b/tests/test_pkgs_dryrun.py
index f936c6b8..971822f5 100644
--- a/tests/test_pkgs_dryrun.py
+++ b/tests/test_pkgs_dryrun.py
@@ -55,7 +55,7 @@ def test_maven_delete_dry_run(self):
test_bucket = self.mock_s3.Bucket(TEST_MVN_BUCKET)
objs = list(test_bucket.objects.all())
- self.assertEqual(32, len(objs))
+ self.assertEqual(36, len(objs))
def test_npm_upload_dry_run(self):
test_tgz = os.path.join(os.getcwd(), "tests/input/code-frame-7.14.5.tgz")