validate config with schema and allow target to be a list of buckets

charon/cmd/command.py

@@ -16,6 +16,7 @@
 from typing import List, Tuple
 
 from charon.config import CharonConfig, get_config
+from charon.constants import DEFAULT_REGISTRY
 from charon.utils.logs import set_logging
 from charon.utils.archive import detect_npm_archive, download_archive, NpmArchiveType
 from charon.pkgs.maven import handle_maven_uploading, handle_maven_del
@@ -60,6 +61,7 @@
 @option(
     "--target",
     "-t",
+    'targets',
     help="""
     The target to do the uploading, which will decide which s3 bucket
     and what root path where all files will be uploaded to.
@@ -114,7 +116,7 @@ def upload(
     repo: str,
     product: str,
     version: str,
-    target: List[str],
+    targets: List[str],
     root_path="maven-repository",
     ignore_patterns: List[str] = None,
     work_dir: str = None,
@@ -147,13 +149,13 @@ def upload(
         npm_archive_type = detect_npm_archive(archive_path)
         product_key = f"{product}-{version}"
         manifest_bucket_name = conf.get_manifest_bucket()
-        targets_ = __get_targets(target, conf)
+        buckets = __get_buckets(targets, conf)
         if npm_archive_type != NpmArchiveType.NOT_NPM:
             logger.info("This is a npm archive")
             tmp_dir, succeeded = handle_npm_uploading(
                 archive_path,
                 product_key,
-                targets=targets_,
+                buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
                 dry_run=dryrun,
@@ -173,7 +175,7 @@ def upload(
                 product_key,
                 ignore_patterns_list,
                 root=root_path,
-                targets=targets_,
+                buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
                 dry_run=dryrun,
@@ -217,6 +219,7 @@ def upload(
 @option(
     "--target",
     "-t",
+    'targets',
     help="""
     The target to do the deletion, which will decide which s3 bucket
     and what root path where all files will be deleted from.
@@ -270,7 +273,7 @@ def delete(
     repo: str,
     product: str,
     version: str,
-    target: List[str],
+    targets: List[str],
     root_path="maven-repository",
     ignore_patterns: List[str] = None,
     work_dir: str = None,
@@ -303,13 +306,13 @@ def delete(
         npm_archive_type = detect_npm_archive(archive_path)
         product_key = f"{product}-{version}"
         manifest_bucket_name = conf.get_manifest_bucket()
-        targets_ = __get_targets(target, conf)
+        buckets = __get_buckets(targets, conf)
         if npm_archive_type != NpmArchiveType.NOT_NPM:
             logger.info("This is a npm archive")
             tmp_dir, succeeded = handle_npm_del(
                 archive_path,
                 product_key,
-                targets=targets_,
+                buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
                 dry_run=dryrun,
@@ -329,7 +332,7 @@ def delete(
                 product_key,
                 ignore_patterns_list,
                 root=root_path,
-                targets=targets_,
+                buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
                 dry_run=dryrun,
@@ -345,22 +348,15 @@ def delete(
             __safe_delete(tmp_dir)
 
 
-def __get_targets(target: List[str], conf: CharonConfig) -> List[Tuple[str, str, str, str]]:
-    targets_ = []
-    for tgt in target:
-        aws_bucket = conf.get_aws_bucket(tgt)
-        if not aws_bucket:
-            continue
-        prefix = conf.get_bucket_prefix(tgt)
-        registry = conf.get_bucket_registry(tgt)
-        targets_.append([tgt, aws_bucket, prefix, registry])
-    if len(targets_) == 0:
-        logger.error(
-            "All targets are not valid or configured, "
-            "please check your charon configurations."
-        )
-        sys.exit(1)
-    return targets_
+def __get_buckets(targets: List[str], conf: CharonConfig) -> List[Tuple[str, str, str, str]]:
+    buckets = []
+    for target in targets:
+        for bucket in conf.get_target(target):
+            aws_bucket = bucket.get('bucket')
+            prefix = bucket.get('prefix', '')
+            registry = bucket.get('registry', DEFAULT_REGISTRY)
+            buckets.append((target, aws_bucket, prefix, registry))
+    return buckets
 
 
 def __safe_delete(tmp_dir: str):

charon/config.py

@@ -13,15 +13,16 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from typing import Dict, List
+import json
+from typing import Dict, List, Optional
+
+import jsonschema
+from jsonschema.exceptions import ValidationError
 from ruamel.yaml import YAML
 from pathlib import Path
 import os
 import logging
 
-from charon.utils.strings import remove_prefix
-from charon.constants import DEFAULT_REGISTRY
-
 CONFIG_FILE = "charon.yaml"
 
 logger = logging.getLogger(__name__)
@@ -37,73 +38,38 @@ def __init__(self, data: Dict):
         self.__ignore_patterns: List[str] = data.get("ignore_patterns", None)
         self.__aws_profile: str = data.get("aws_profile", None)
         self.__targets: Dict = data.get("targets", None)
-        if not self.__targets or not isinstance(self.__targets, Dict):
-            raise TypeError("Charon configuration is not correct: targets is invalid.")
         self.__manifest_bucket: str = data.get("manifest_bucket", None)
 
     def get_ignore_patterns(self) -> List[str]:
         return self.__ignore_patterns
 
+    def get_target(self, target: str) -> List[Dict]:
+        target_: List = self.__targets.get(target, [])
+        if not target_:
+            logger.error("The target %s is not found in charon configuration.", target)
+        return target_
+
     def get_aws_profile(self) -> str:
         return self.__aws_profile
 
-    def get_aws_bucket(self, target: str) -> str:
-        target_: Dict = self.__targets.get(target, None)
-        if not target_ or not isinstance(target_, Dict):
-            logger.error("The target %s is not found in charon configuration.", target)
-            return None
-        bucket = target_.get("bucket", None)
-        if not bucket:
-            logger.error("The bucket is not found for target %s "
-                         "in charon configuration.", target)
-        return bucket
-
-    def get_bucket_prefix(self, target: str) -> str:
-        target_: Dict = self.__targets.get(target, None)
-        if not target_ or not isinstance(target_, Dict):
-            logger.error("The target %s is not found in charon "
-                         "configuration.", target)
-            return None
-        prefix = target_.get("prefix", None)
-        if not prefix:
-            logger.warning("The prefix is not found for target %s "
-                           "in charon configuration, so no prefix will "
-                           "be used", target)
-            prefix = ""
-        # removing first slash as it is not needed.
-        prefix = remove_prefix(prefix, "/")
-        return prefix
-
-    def get_bucket_registry(self, target: str) -> str:
-        target_: Dict = self.__targets.get(target, None)
-        if not target_ or not isinstance(target_, Dict):
-            logger.error("The target %s is not found in charon configuration.", target)
-            return None
-        registry = target_.get("registry", None)
-        if not registry:
-            registry = DEFAULT_REGISTRY
-            logger.error("The registry is not found for target %s "
-                         "in charon configuration, so DEFAULT_REGISTRY(localhost) will be used.",
-                         target)
-        return registry
-
     def get_manifest_bucket(self) -> str:
         return self.__manifest_bucket
 
 
-def get_config() -> CharonConfig:
+def get_config() -> Optional[CharonConfig]:
     config_file = os.path.join(os.getenv("HOME"), ".charon", CONFIG_FILE)
     try:
         yaml = YAML(typ='safe')
         data = yaml.load(stream=Path(config_file))
-    except Exception as e:
-        logger.error("Can not load charon config file due to error: %s", e)
-        return None
-    try:
-        return CharonConfig(data)
-    except TypeError as e:
-        logger.error(e)
-        return None
+        with open(os.path.join(os.getcwd(), 'charon', 'schemas', 'charon.json'), 'r') as f:
+            schema = json.load(f)
+        validator = jsonschema.Draft7Validator(schema=schema)
+        jsonschema.Draft7Validator.check_schema(schema)
+        validator.validate(data)
+    except ValidationError as e:
+        logger.error("Invalid configuration: %s", e)
+        raise e
+    return CharonConfig(data)
 
 
 def get_template(template_file: str) -> str: