[merge from main] Release charon 1.3.1

charon.spec

@@ -80,6 +80,11 @@ export LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8
 
 
 %changelog
+* Tue May 7 2024 Gang Li <gli@redhat.com>
+- 1.3.1 release
+- Add checksum refresh command: refresh checksum files for maven artifacts
+- Refactor the CF invalidating commands into cf sub command
+
 * Fri Apr 12 2024 Gang Li <gli@redhat.com>
 - 1.3.0 release
 - Add validate command: validate the checksum for maven artifacts

charon/cmd/__init__.py

@@ -17,8 +17,8 @@
 from charon.cmd.cmd_upload import upload
 from charon.cmd.cmd_delete import delete
 from charon.cmd.cmd_index import index
-from charon.cmd.cmd_checksum import checksum_validate
-from charon.cmd.cmd_cache import cf_invalidate, cf_check
+from charon.cmd.cmd_checksum import init_checksum, checksum
+from charon.cmd.cmd_cache import init_cf, cf
 
 
 @group()
@@ -33,6 +33,11 @@ def cli():
 cli.add_command(upload)
 cli.add_command(delete)
 cli.add_command(index)
-cli.add_command(checksum_validate)
-cli.add_command(cf_invalidate)
-cli.add_command(cf_check)
+
+# init cf cmmand
+init_cf()
+cli.add_command(cf)
+
+# init checksum command
+init_checksum()
+cli.add_command(checksum)

charon/cmd/cmd_cache.py

@@ -18,7 +18,7 @@
 from charon.cmd.internal import _decide_mode, _get_buckets
 from charon.cache import CFClient
 from charon.pkgs.pkg_utils import invalidate_cf_paths
-from click import command, option, argument
+from click import command, option, argument, group
 from typing import List, Tuple
 
 import traceback
@@ -54,7 +54,7 @@
     "-f",
     "path_file",
     help="""
-    The file which contain the paths to be invalidated in CF. Pahts in this file follow the
+    The file which contain the paths to be invalidated in CF. Paths in this file follow the
     format of CF defining too, and each path should be in a single line.
     """
 )
@@ -75,7 +75,7 @@
     default=False
 )
 @command()
-def cf_invalidate(
+def invalidate(
     target: str,
     paths: List[str],
     path_file: str,
@@ -161,7 +161,7 @@ def cf_invalidate(
     default=False
 )
 @command()
-def cf_check(
+def check(
     invalidation_id: str,
     target: str,
     quiet: bool = False,
@@ -214,3 +214,15 @@ def _init_cmd(target: str) -> Tuple[List[Tuple[str, str, str, str, str]], str]:
         sys.exit(1)
 
     return (_get_buckets([target], conf), aws_profile)
+
+
+@group()
+def cf():
+    """cf commands are responsible for the CloudFront cache operations in
+    products operated by Charon
+    """
+
+
+def init_cf():
+    cf.add_command(invalidate)
+    cf.add_command(check)

charon/cmd/cmd_checksum.py

@@ -13,12 +13,14 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from typing import List
+from typing import List, Tuple
 
 from charon.config import get_config
-from charon.pkgs.checksum_http import handle_checksum_validation_http
+from charon.pkgs.checksum_http import (
+    handle_checksum_validation_http, refresh_checksum
+)
 from charon.cmd.internal import _decide_mode
-from click import command, option, argument
+from click import command, option, argument, group
 
 import traceback
 import logging
@@ -99,7 +101,7 @@
     required=True
 )
 @command()
-def checksum_validate(
+def validate(
     path: str,
     target: str,
     includes: List[str],
@@ -118,22 +120,11 @@ def checksum_validate(
     be recorded.
     """
     _decide_mode(
-        "checksum-{}".format(target), path.replace("/", "_"),
+        "checksum-validate-{}".format(target), path.replace("/", "_"),
         is_quiet=quiet, is_debug=debug
     )
     try:
-        conf = get_config()
-        if not conf:
-            sys.exit(1)
-
-        aws_bucket = ""
-        root_path = ""
-        t = conf.get_target(target)
-        if not t:
-            sys.exit(1)
-        for b in t:
-            aws_bucket = b.get('bucket')
-            prefix = b.get('prefix', '')
+        (aws_bucket, prefix) = _init_cmd(target)
 
         # NOTE: This is a liitle hacky, which constrain the configuration of
         #       of target should define the bucket to contain "prod-maven"
@@ -153,3 +144,129 @@ def checksum_validate(
     except Exception:
         print(traceback.format_exc())
         sys.exit(2)
+
+
+@option(
+    "--debug",
+    "-D",
+    "debug",
+    help="Debug mode, will print all debug logs for problem tracking.",
+    is_flag=True,
+    default=False
+)
+@option(
+    "--quiet",
+    "-q",
+    "quiet",
+    help="Quiet mode, will shrink most of the logs except warning and errors.",
+    is_flag=True,
+    default=False
+)
+@option(
+    "--path",
+    "-p",
+    "paths",
+    help="""
+    The paths of artifact files to do checksum refreshing.
+    """,
+    multiple=True
+)
+@option(
+    "--path-file",
+    "-f",
+    "path_file",
+    help="""
+    The file which contain the paths of artifact files to do checksum refreshing.
+    Each path in this file should be in a single line.
+    """
+)
+@option(
+    "--target",
+    "-t",
+    "target",
+    help="""
+    The target to do the uploading, which will decide which s3 bucket
+    and what root path where all files will be uploaded to.
+    Can accept more than one target.
+    """,
+    required=True
+)
+@command()
+def refresh(
+    target: str,
+    paths: List[str],
+    path_file: str,
+    quiet: bool = False,
+    debug: bool = False
+):
+    """
+    Refresh the checksum of the specified path for the target maven repository.
+    It will calculate the checksum files of the specified artifact and see if
+    unmatched, then regenerate the checksum files based on the artifact.
+    Default checksum files include .md5, .sha1.
+    """
+    _decide_mode(
+        "checksum-refresh-{}".format(target), "",
+        is_quiet=quiet, is_debug=debug, use_log_file=False
+    )
+    if not paths and not path_file:
+        logger.error(
+            "No path specified, please specify at least one path "
+            "through --path or --path-file.")
+        sys.exit(1)
+
+    work_paths = []
+    if paths:
+        work_paths.extend(paths)
+
+    conf = get_config()
+    aws_profile = os.getenv("AWS_PROFILE") or conf.get_aws_profile()
+    if not aws_profile:
+        logger.error("No AWS profile specified!")
+        sys.exit(1)
+
+    if path_file:
+        with open(path_file, "r", encoding="utf-8") as f:
+            for line in f.readlines():
+                work_paths.append(str(line).strip())
+    try:
+        (aws_bucket, prefix) = _init_cmd(target)
+
+        # NOTE: This is a liitle hacky, which constrain the configuration of
+        #       of target should define the bucket to contain "prod-maven"
+        #       or "stage-maven" to decide that the bucket is for maven repo
+        #       in our defined aws env for production or stage
+        if "prod-maven" not in aws_bucket and "stage-maven" not in aws_bucket:
+            logger.error("The target %s is not a maven repository.", target)
+            sys.exit(1)
+
+        refresh_checksum((aws_bucket, prefix), work_paths, aws_profile)
+    except Exception:
+        print(traceback.format_exc())
+        sys.exit(2)
+
+
+def _init_cmd(target: str) -> Tuple[str, str]:
+    conf = get_config()
+    if not conf:
+        sys.exit(1)
+    aws_bucket = ""
+    t = conf.get_target(target)
+    if not t:
+        sys.exit(1)
+    for b in t:
+        aws_bucket = b.get('bucket')
+        prefix = b.get('prefix', '')
+    return (aws_bucket, prefix)
+
+
+@group()
+def checksum():
+    """checksum commands are responsible to operate checksum files
+    of maven products operated by Charon
+    """
+
+
+def init_checksum():
+    checksum.add_command(validate)
+    checksum.add_command(refresh)

charon/pkgs/checksum_http.py

@@ -13,7 +13,8 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from charon.utils.files import digest
+from charon.utils.files import digest, HashType
+from charon.storage import S3Client
 from typing import Tuple, List, Dict
 from html.parser import HTMLParser
 import tempfile
@@ -36,9 +37,8 @@ def handle_checksum_validation_http(
     skips: List[str] = None
 ):
     """ Handle the checksum check for maven artifacts.
-        * target contains bucket name and prefix for the bucket, which will
-          be used to store artifacts with the prefix. See target definition
-          in Charon configuration for details.
+        * bucket contains store artifacts with the prefix. See target
+          definition in Charon configuration for details.
         * path is the root path where to start the validation in the bucket.
         * includes are the file suffixes which will decide the types of files
           to do the validation.
@@ -266,3 +266,84 @@ def _decide_root_url(bucket: str) -> str:
     if bucket.strip().startswith("stage-maven"):
         return "https://maven.stage.repository.redhat.com"
     return None
+
+
+def refresh_checksum(
+    target: Tuple[str, str],
+    paths: List[str],
+    aws_profile: str = None
+):
+    """Refresh checksum for files in a given bucket.
+       * bucket contains store artifacts with the prefix. See target
+         definition in Charon configuration for details.
+       * paths are the exact files whose checksum files will be
+         refreshed with.
+    """
+    bucket_name = target[0]
+    prefix = target[1]
+    s3_client = S3Client(aws_profile=aws_profile)
+    real_prefix = prefix if prefix.strip() != "/" else ""
+    filetype_filter = [".prodinfo", ".sha1", ".sha256", ".md5"]
+    for path in paths:
+        is_artifact = True
+        for filetype in filetype_filter:
+            if path.strip().endswith(filetype):
+                is_artifact = False
+                continue
+        if not is_artifact:
+            logger.info(
+                "%s is not an artifact file for maven products. Skipped.",
+                path
+            )
+            continue
+        s3_path = os.path.join(real_prefix, path)
+        checksums = {
+            ".md5": HashType.MD5,
+            ".sha1": HashType.SHA1,
+            ".sha256": HashType.SHA256,
+            ".sha512": HashType.SHA512
+        }
+        if s3_client.file_exists_in_bucket(bucket_name, s3_path):
+            temp_f = os.path.join(tempfile.gettempdir(), path)
+            folder = os.path.dirname(temp_f)
+            try:
+                if not os.path.exists(folder):
+                    os.makedirs(folder)
+                s3_client.download_file(bucket_name, s3_path, temp_f)
+                existed_checksum_types = []
+                for file_type in checksums:
+                    s3_checksum_path = s3_path + file_type
+                    if s3_client.file_exists_in_bucket(bucket_name, s3_checksum_path):
+                        existed_checksum_types.append(file_type)
+                if existed_checksum_types:
+                    for file_type in existed_checksum_types:
+                        checksum_path = path + file_type
+                        s3_checksum_path = s3_path + file_type
+                        hash_type = checksums[file_type]
+                        correct_checksum_c = digest(temp_f, hash_type)
+                        original_checksum_c = s3_client.read_file_content(
+                            bucket_name, s3_checksum_path
+                        )
+                        if correct_checksum_c == original_checksum_c:
+                            logger.info("Checksum %s matches, no need to refresh.", checksum_path)
+                        else:
+                            logger.info("Checksum %s does not match, refreshing...", checksum_path)
+                            s3_client.simple_upload_file(
+                                file_path=checksum_path,
+                                file_content=correct_checksum_c,
+                                target=(bucket_name, prefix),
+                                mime_type="text/plain",
+                                force=True
+                            )
+                else:
+                    logger.warning(
+                        "No valid checksum files exist for %s, Skipped."
+                        " Are you sure it is a valid maven artifact?",
+                        path
+                    )
+            finally:
+                if folder and folder != tempfile.gettempdir() and os.path.exists(folder):
+                    shutil.rmtree(folder)
+            logger.info("Checksums are refreshed for artifact %s", path)
+        else:
+            logger.warning("File %s does not exist in bucket %s", s3_path, bucket_name)

charon/pkgs/pkg_utils.py

@@ -117,7 +117,7 @@ def invalidate_cf_paths(
                         non_completed[status] = ids
                 logger.info(
                     "The CF invalidating requests done, following requests "
-                    "are not completed yet:\n %s\nPlease use cf-check command to "
+                    "are not completed yet:\n %s\nPlease use 'cf check' command to "
                     "check its details.", non_completed
                 )
                 logger.debug(