Merge from 1.3.x

charon/cmd/cmd_index.py

@@ -42,6 +42,13 @@
     """,
     required=True
 )
+@option(
+    "--recursive",
+    "-r",
+    help="If do indexing recursively under $path",
+    is_flag=True,
+    default=False
+)
 @option(
     "--config",
     "-c",
@@ -69,6 +76,7 @@
 def index(
     path: str,
     target: str,
+    recursive: bool = False,
     config: str = None,
     debug: bool = False,
     quiet: bool = False,
@@ -120,7 +128,15 @@ def index(
             if not aws_bucket:
                 logger.error("No bucket specified for target %s!", target)
             else:
-                re_index(b, path, package_type, aws_profile, dryrun)
+                args = {
+                    "target": b,
+                    "path": path,
+                    "package_type": package_type,
+                    "aws_profile": aws_profile,
+                    "recursive": recursive,
+                    "dry_run": dryrun
+                }
+                re_index(**args)  # type: ignore
 
     except Exception:
         print(traceback.format_exc())

charon/cmd/cmd_upload.py

@@ -16,12 +16,12 @@
 from typing import List
 
 from charon.config import get_config
-from charon.utils.archive import detect_npm_archive, NpmArchiveType
+from charon.utils.archive import detect_npm_archives, NpmArchiveType
 from charon.pkgs.maven import handle_maven_uploading
 from charon.pkgs.npm import handle_npm_uploading
 from charon.cmd.internal import (
     _decide_mode, _validate_prod_key,
-    _get_local_repo, _get_targets,
+    _get_local_repos, _get_targets,
     _get_ignore_patterns, _safe_delete
 )
 from click import command, option, argument
@@ -35,8 +35,9 @@
 
 
 @argument(
-    "repo",
+    "repos",
     type=str,
+    nargs=-1  # This allows multiple arguments for zip urls
 )
 @option(
     "--product",
@@ -146,7 +147,7 @@
 )
 @command()
 def upload(
-    repo: str,
+    repos: List[str],
     product: str,
     version: str,
     targets: List[str],
@@ -161,9 +162,10 @@ def upload(
     dryrun=False,
     sign_result_file=None,
 ):
-    """Upload all files from a released product REPO to Ronda
-    Service. The REPO points to a product released tarball which
-    is hosted in a remote url or a local path.
+    """Upload all files from released product REPOs to Ronda
+    Service. The REPOs point to a product released tarballs which
+    are hosted in remote urls or local paths.
+    Notes: It does not support multiple repos for NPM archives
     """
     tmp_dir = work_dir
     try:
@@ -182,8 +184,8 @@ def upload(
             logger.error("No AWS profile specified!")
             sys.exit(1)
 
-        archive_path = _get_local_repo(repo)
-        npm_archive_type = detect_npm_archive(archive_path)
+        archive_paths = _get_local_repos(repos)
+        archive_types = detect_npm_archives(archive_paths)
         product_key = f"{product}-{version}"
         manifest_bucket_name = conf.get_manifest_bucket()
         targets_ = _get_targets(targets, conf)
@@ -194,31 +196,18 @@ def upload(
                 " are set correctly.", targets_
             )
             sys.exit(1)
-        if npm_archive_type != NpmArchiveType.NOT_NPM:
-            logger.info("This is a npm archive")
-            tmp_dir, succeeded = handle_npm_uploading(
-                archive_path,
-                product_key,
-                targets=targets_,
-                aws_profile=aws_profile,
-                dir_=work_dir,
-                gen_sign=contain_signature,
-                cf_enable=conf.is_aws_cf_enable(),
-                key=sign_key,
-                dry_run=dryrun,
-                manifest_bucket_name=manifest_bucket_name
-            )
-            if not succeeded:
-                sys.exit(1)
-        else:
+
+        maven_count = archive_types.count(NpmArchiveType.NOT_NPM)
+        npm_count = len(archive_types) - maven_count
+        if maven_count == len(archive_types):
             ignore_patterns_list = None
             if ignore_patterns:
                 ignore_patterns_list = ignore_patterns
             else:
                 ignore_patterns_list = _get_ignore_patterns(conf)
             logger.info("This is a maven archive")
             tmp_dir, succeeded = handle_maven_uploading(
-                archive_path,
+                archive_paths,
                 product_key,
                 ignore_patterns_list,
                 root=root_path,
@@ -235,6 +224,28 @@ def upload(
             )
             if not succeeded:
                 sys.exit(1)
+        elif npm_count == len(archive_types) and len(archive_types) == 1:
+            logger.info("This is a npm archive")
+            tmp_dir, succeeded = handle_npm_uploading(
+                archive_paths[0],
+                product_key,
+                targets=targets_,
+                aws_profile=aws_profile,
+                dir_=work_dir,
+                gen_sign=contain_signature,
+                cf_enable=conf.is_aws_cf_enable(),
+                key=sign_key,
+                dry_run=dryrun,
+                manifest_bucket_name=manifest_bucket_name
+            )
+            if not succeeded:
+                sys.exit(1)
+        elif npm_count == len(archive_types) and len(archive_types) > 1:
+            logger.error("Doesn't support multiple upload for npm")
+            sys.exit(1)
+        else:
+            logger.error("Upload types are not consistent")
+            sys.exit(1)
     except Exception:
         print(traceback.format_exc())
         sys.exit(2)  # distinguish between exception and bad config or bad state

charon/cmd/internal.py

@@ -75,6 +75,14 @@ def _get_local_repo(url: str) -> str:
     return archive_path
 
 
+def _get_local_repos(urls: list) -> list:
+    archive_paths = []
+    for url in urls:
+        archive_path = _get_local_repo(url)
+        archive_paths.append(archive_path)
+    return archive_paths
+
+
 def _validate_prod_key(product: str, version: str) -> bool:
     if not product or product.strip() == "":
         logger.error("Error: product can not be empty!")

charon/pkgs/checksum_http.py

@@ -13,7 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from charon.utils.files import digest, HashType
+from charon.utils.files import digest, HashType, overwrite_file
 from charon.storage import S3Client
 from typing import Tuple, List, Dict, Optional
 from html.parser import HTMLParser
@@ -169,9 +169,10 @@ def _check_and_remove_file(file_name: str):
     def _write_one_col_file(items: List[str], file_name: str):
         if items and len(items) > 0:
             _check_and_remove_file(file_name)
-            with open(file_name, "w") as f:
-                for i in items:
-                    f.write(i + "\n")
+            content = ""
+            for i in items:
+                content = content + i + "\n"
+            overwrite_file(file_name, content)
             logger.info("The report file %s is generated.", file_name)
 
     _write_one_col_file(content[0], os.path.join(work_dir, "mismatched_files.csv"))
@@ -180,10 +181,9 @@ def _write_one_col_file(items: List[str], file_name: str):
     if content[2] and len(content[2]) > 0:
         error_file = os.path.join(work_dir, "error_files.csv")
         _check_and_remove_file(error_file)
-        with open(error_file, "w") as f:
-            f.write("path,error\n")
-            for d in content[2]:
-                f.write("{path},{error}\n".format(path=d["path"], error=d["error"]))
+        f_content_lines: List[str] = []
+        f_content = "path,error\n" + "\n".join(f_content_lines)
+        overwrite_file(error_file, f_content)
         logger.info("The report file %s is generated.", error_file)
 
 

charon/pkgs/indexing.py

@@ -19,7 +19,7 @@
 # from charon.pkgs.pkg_utils import invalidate_cf_paths
 from charon.constants import (INDEX_HTML_TEMPLATE, NPM_INDEX_HTML_TEMPLATE,
                               PACKAGE_TYPE_MAVEN, PACKAGE_TYPE_NPM, PROD_INFO_SUFFIX)
-from charon.utils.files import digest_content
+from charon.utils.files import digest_content, overwrite_file
 from jinja2 import Template
 import os
 import logging
@@ -155,8 +155,7 @@ def __to_html(package_type: str, contents: List[str], folder: str, top_level: st
     if folder == "/":
         html_path = os.path.join(top_level, "index.html")
     os.makedirs(os.path.dirname(html_path), exist_ok=True)
-    with open(html_path, 'w', encoding='utf-8') as html:
-        html.write(html_content)
+    overwrite_file(html_path, html_content)
     return html_path
 
 
@@ -267,7 +266,7 @@ def re_index(
     path: str,
     package_type: str,
     aws_profile: str = None,
-    # cf_enable: bool = False,
+    recursive: bool = False,
     dry_run: bool = False
 ):
     """Refresh the index.html for the specified folder in the bucket.
@@ -307,17 +306,31 @@ def re_index(
         logger.debug("The re-indexed page content: %s", index_content)
         if not dry_run:
             index_path = os.path.join(path, "index.html")
+            logger.info("Start re-indexing %s in bucket %s", index_path, bucket_name)
             if path == "/":
                 index_path = "index.html"
             s3_client.simple_delete_file(index_path, (bucket_name, real_prefix))
             s3_client.simple_upload_file(
                 index_path, index_content, (bucket_name, real_prefix),
                 "text/html", digest_content(index_content)
             )
-            # We will not invalidate index.html per cost consideration
-            # if cf_enable:
-            #     cf_client = CFClient(aws_profile=aws_profile)
-            #     invalidate_cf_paths(cf_client, bucket, [index_path])
+            logger.info("%s re-indexing finished", index_path)
+        if recursive:
+            for c in contents:
+                if c.endswith("/"):
+                    sub_path = c.removeprefix(real_prefix).strip()
+                    if sub_path.startswith("/"):
+                        sub_path = sub_path.removeprefix("/")
+                    logger.debug("subpath: %s", sub_path)
+                    args = {
+                        "target": target,
+                        "path": sub_path,
+                        "package_type": package_type,
+                        "aws_profile": aws_profile,
+                        "recursive": recursive,
+                        "dry_run": dry_run
+                    }
+                    re_index(**args)  # type: ignore
     else:
         logger.warning(
             "The path %s does not contain any contents in bucket %s. "