Merge pull request #535 from JrGoodle/tests

Test updates

script/update

@@ -8,7 +8,6 @@ if [ "$1" == 'version' ]; then
     NEW_VERSION=$2
 
     perl -pi -e "s/$OLD_VERSION/$NEW_VERSION/g" src/setup.py || exit 1
-    perl -pi -e "s/$OLD_VERSION/$NEW_VERSION/g" src/clowder/clowder_app.py || exit 1
     perl -pi -e "s/$OLD_VERSION/$NEW_VERSION/g" .github/issue_template.md || exit 1
     perl -pi -e "s/$OLD_VERSION/$NEW_VERSION/g" docs/conf.py || exit 1
     exit

src/clowder/__init__.py

@@ -12,7 +12,7 @@
 from typing import Optional
 
 from clowder.error import ClowderExit
-from clowder.util.formatting import ERROR
+from clowder.util.formatting import error_ambiguous_clowder_yaml
 from clowder.git.util import existing_git_repository
 
 # Set up logging #
@@ -31,7 +31,6 @@ def LOG_DEBUG(message: str, exception: Optional[Exception] = None): # noqa
     if PRINT_DEBUG_OUTPUT:
         logger.log(logging.DEBUG, f" {message}")
         if exception is not None:
-            logger.log(logging.DEBUG, str(exception))
             # TODO: Format the output for clowder debug
             traceback.print_exc()
 
@@ -75,11 +74,12 @@ def existing_clowder_repo(directory: Path) -> bool:
     clowder_yml = CLOWDER_DIR / 'clowder.yml'
     clowder_yaml = CLOWDER_DIR / 'clowder.yaml'
     if clowder_yml.is_symlink() and clowder_yaml.is_symlink():
-        print(f"\n{ERROR} Found clowder.yml and clowder.yaml files in same directory\n")
+        print(error_ambiguous_clowder_yaml())
         try:
             raise ClowderExit(1)
         except ClowderExit as err:
             LOG_DEBUG('Ambigiuous clowder file', err)
+            print()
             exit(err.code)
     if clowder_yml.is_symlink():
         CLOWDER_YAML = clowder_yml

src/clowder/clowder_app.py

@@ -6,31 +6,26 @@
 """
 
 import argparse
+import pkg_resources
 from multiprocessing import freeze_support
-from typing import List, Tuple
 
 # noinspection PyPackageRequirements
 import argcomplete
 import colorama
 
 import clowder.cli as cmd
 from clowder import LOG_DEBUG
-from clowder.error import ClowderExit
-from clowder.util.parallel import __clowder_pool__
-
-VERSION = '3.2.0'
+from clowder.error import ClowderConfigYAMLError, ClowderExit
 
 
 class ClowderArgumentParser(argparse.ArgumentParser):
 
     def error(self, message):
         # Make sure mp pool is closed
-        close_mp_pool()
         argparse.ArgumentParser.error(self, message)
 
     def exit(self, status=0, message=None):
         # Make sure mp pool is closed
-        close_mp_pool()
         if message is not None:
             message = f"{message}\n"
         else:
@@ -40,8 +35,9 @@ def exit(self, status=0, message=None):
 
 def create_parsers() -> ClowderArgumentParser:
     parser = ClowderArgumentParser(prog='clowder')
+    version_message = f"clowder version {pkg_resources.require('clowder-repo')[0].version}"
     arguments = [
-        (['-v', '--version'], dict(action='version', version=VERSION))
+        (['-v', '--version'], dict(action='version', version=version_message))
     ]
     cmd.add_parser_arguments(parser, arguments)
     subparsers = parser.add_subparsers(help='sub-command help')
@@ -71,46 +67,34 @@ def main() -> None:
     """Clowder command CLI main function"""
 
     print()
-    exit_code = 0
-
-    parser = create_parsers()
-    argcomplete.autocomplete(parser)
-    args = parser.parse_args()
-    if 'projects' in args:
-        if isinstance(args.projects, str):
-            args.projects = [args.projects]
+    parser = None
     try:
+        parser = create_parsers()
+        argcomplete.autocomplete(parser)
+        args = parser.parse_args()
+        if 'projects' in args:
+            if isinstance(args.projects, str):
+                args.projects = [args.projects]
         args.func(args) # noqa
     except ClowderExit as err:
         LOG_DEBUG('ClowderExit exception', err)
-        exit_code = err.code
+        print()
+        exit(err.code)
+    except ClowderConfigYAMLError as err:
+        print(err.message)
+        LOG_DEBUG('ClowderConfigYAMLError exception', err)
+        print()
+        exit(err.code)
     except AttributeError as err:
         LOG_DEBUG('AttributeError exception', err)
-        exit_code = 1
         if parser is not None:
             parser.print_help()
+        print()
+        exit(1)
     except Exception as err: # noqa
         LOG_DEBUG('Unhandled generic exception', err)
-        exit_code = 1
-    finally:
         print()
-
-        # Make sure mp pool is closed
-        close_mp_pool()
-
-        exit(exit_code)
-
-
-def close_mp_pool():
-    """Helper function to close multiprocessing pool"""
-    if __clowder_pool__ is None:
-        return
-
-    try:
-        __clowder_pool__.close()
-        __clowder_pool__.join()
-    except:  # noqa
-        __clowder_pool__.terminate()
+        exit(1)
 
 
 if __name__ == '__main__':

src/clowder/config/clowder_config.py

@@ -72,7 +72,8 @@ def __init__(self, clowder_config: Optional[dict] = None,
             projects = defaults.get('projects', None)
             self.projects: Optional[Tuple[str, ...]] = None if projects is None else tuple(sorted(projects))
             # TODO: Confirm projects exist, otherwise throw error, maybe offer to clean up?
-            self.validate_config_projects_defined(self.projects, project_options)
+            if self.projects is not None:
+                self.validate_config_projects_defined(self.projects, project_options)
             self.protocol: Optional[str] = defaults.get('protocol', None)
             self.rebase: Optional[bool] = defaults.get('rebase', None)
             self.parallel: Optional[bool] = defaults.get('parallel', None)

src/clowder/config/config.py

@@ -9,7 +9,7 @@
 
 import clowder.util.formatting as fmt
 from clowder import CLOWDER_DIR, CLOWDER_CONFIG_DIR, CLOWDER_CONFIG_YAML
-from clowder.error import ClowderConfigYAMLError, ClowderExit
+from clowder.error import ClowderConfigYAMLError, ClowderConfigYAMLErrorType, ClowderExit
 from clowder.util.file_system import (
     create_backup_file,
     make_dir,
@@ -40,6 +40,8 @@ def __init__(self, current_clowder_name: Optional[str], project_options: Tuple[s
         """Config __init__"""
 
         self.error = None
+        self.current_clowder_config = None
+        self.clowder_configs = ()
         self._project_options = project_options
 
         # Config file doesn't currently exist
@@ -52,37 +54,21 @@ def __init__(self, current_clowder_name: Optional[str], project_options: Tuple[s
             return
 
         # Config file does exist, try to load
-        try:
-            self._load_clowder_config_yaml()
-        except ClowderConfigYAMLError as err:
-            self.error = err
-        except Exception as err: # noqa
-            self.error = err
-            # TODO: Remove file?
-        except (KeyboardInterrupt, SystemExit):
-            raise ClowderExit(1)
+        self._load_clowder_config_yaml()
 
-        # If current clowder config doesn't exist, create empty one
+        # If current clowder exists, return
         if self.current_clowder_config is not None:
-            if self.error:
-                raise ClowderExit(self.error.code)
             return
 
         # If current clowder config doesn't exist, create empty one
-        try:
-            self.current_clowder_config = ClowderConfig(current_clowder_name=current_clowder_name,
-                                                        project_options=self._project_options)
-        except ClowderConfigYAMLError as err:
-            self.error = err
-        except (KeyboardInterrupt, SystemExit):
-            raise ClowderExit(1)
+        self.current_clowder_config = ClowderConfig(current_clowder_name=current_clowder_name,
+                                                    project_options=self._project_options)
+        if not self.clowder_configs:
+            self.clowder_configs = (self.current_clowder_config,)
         else:
-            if not self.clowder_configs:
-                self.clowder_configs = (self.current_clowder_config,)
-            else:
-                configs = list(self.clowder_configs)
-                configs.append(self.current_clowder_config)
-                self.clowder_configs = tuple(configs)
+            configs = list(self.clowder_configs)
+            configs.append(self.current_clowder_config)
+            self.clowder_configs = tuple(configs)
 
     def process_projects_arg(self, projects: List[str]) -> Tuple[str, ...]:
         """Process project args based on parameters and config
@@ -159,16 +145,15 @@ def _load_clowder_config_yaml(self) -> None:
                                                      project_options=self._project_options))
             self.clowder_configs = tuple(clowder_configs)
 
-            self.current_clowder_config = None
             if CLOWDER_DIR is not None:
                 for config in self.clowder_configs:
                     if config.clowder_dir.resolve() == CLOWDER_DIR.resolve():
                         self.current_clowder_config = config
                         break
-        except (AttributeError, KeyError, TypeError) as err:
+        except (AttributeError, KeyError, TypeError):
             self.version = CONFIG_VERSION
             self.clowder_configs = ()
             self.current_clowder_config = None
-            self.error = err
+            raise ClowderConfigYAMLError(f"{fmt.ERROR} Invalid config yaml file", ClowderConfigYAMLErrorType.UNKNOWN)
         except (KeyboardInterrupt, SystemExit):
             raise ClowderExit(1)

src/clowder/error/clowder_config_yaml_error.py

@@ -44,5 +44,6 @@ class ClowderConfigYAMLErrorType(IntEnum):
     UNKNOWN = 31
     EMPTY_FILE = 32
     OPEN_FILE = 33
-    MISSING_PROJECT = 34
-    INVALID_CLOWDER_PATH = 35
+    JSONSCHEMA_VALIDATION_FAILED = 34
+    MISSING_PROJECT = 35
+    INVALID_CLOWDER_PATH = 36

src/clowder/util/clowder.config.schema.json

@@ -13,6 +13,11 @@
         "$ref": "#/definitions/nonEmptyString"
       }
     },
+    "projectPath": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^\/.+"
+    },
     "stringArrayUnique": {
       "type": "array",
       "uniqueItems": true,
@@ -86,7 +91,7 @@
           "description": "Path to clowder directory",
           "oneOf": [
             {
-              "$ref": "#/definitions/nonEmptyString"
+              "$ref": "#/definitions/projectPath"
             }
           ]
         },
@@ -104,7 +109,8 @@
   "type": "object",
   "additionalProperties": false,
   "required": [
-    "version"
+    "version",
+    "clowder_configs"
   ],
   "properties": {
     "version": {

src/clowder/util/decorators.py

@@ -34,7 +34,8 @@ def print_clowder_name(func):
     def wrapper(*args, **kwargs):
         """Wrapper"""
 
-        print(fmt.clowder_name(f"{CLOWDER_CONTROLLER.name}\n"))
+        if CLOWDER_CONTROLLER.name is not None:
+            print(fmt.clowder_name(f"{CLOWDER_CONTROLLER.name}\n"))
         return func(*args, **kwargs)
 
     return wrapper

src/clowder/util/formatting.py

@@ -79,7 +79,19 @@ def error(err: Exception) -> str:
     :rtype: str
     """
 
-    return f"{ERROR} {str(err)}\n"
+    return f"{ERROR} {str(err)}"
+
+
+def error_ambiguous_clowder_yaml() -> str:
+    """Return formatted error string for ambiguous clowder yaml file
+
+    :return: Formatted ambigious clowder yaml error
+    :rtype: str
+    """
+
+    yml_file = _yaml_file(Path('clowder.yml'))
+    yaml_file = _yaml_file(Path('clowder.yaml'))
+    return f"\n{ERROR} Found {yml_file} and {yaml_file} files in same directory"
 
 
 def error_command_failed(cmd: Union[str, List[str]]) -> str:
@@ -90,7 +102,7 @@ def error_command_failed(cmd: Union[str, List[str]]) -> str:
     :rtype: str
     """
 
-    return f"{ERROR} Failed to run command {command(cmd)}\n"
+    return f"{ERROR} Failed to run command {command(cmd)}"
 
 
 def error_duplicate_version(version: str) -> str:
@@ -101,7 +113,7 @@ def error_duplicate_version(version: str) -> str:
     :rtype: str
     """
 
-    return f"{ERROR} Duplicate version found: {_yaml_file(Path(version))}\n"
+    return f"{ERROR} Duplicate version found: {_yaml_file(Path(version))}"
 
 
 def error_duplicate_project_path(path: Path, yml: Path) -> str:

src/clowder/util/yaml.py

@@ -216,7 +216,7 @@ def validate_clowder_config_yaml(parsed_yaml: dict) -> None:
         jsonschema.validate(parsed_yaml, json_schema)
     except jsonschema.exceptions.ValidationError as err:
         error_message = f"{fmt.error_invalid_yaml_file(CLOWDER_CONFIG_YAML.name)}\n{fmt.ERROR} {err.message}"
-        raise ClowderYAMLError(error_message, ClowderYAMLErrorType.JSONSCHEMA_VALIDATION_FAILED)
+        raise ClowderConfigYAMLError(error_message, ClowderConfigYAMLErrorType.JSONSCHEMA_VALIDATION_FAILED)
 
 
 def validate_clowder_yaml(parsed_yaml: dict) -> None:

test/config/v0.1/invalid/test-arg-value-clowder_dir.clowder.config.yml

@@ -0,0 +1,9 @@
+version: 0.1
+
+clowder_configs:
+  - name: cats
+    clowder_dir: Users/jrgoodle/projects/cats # ERROR
+    defaults:
+      parallel: true
+      projects: [kit, kishka]
+

test/config/v0.1/invalid/test-empty.clowder.config.yml

@@ -0,0 +1 @@
+ # ERROR

test/config/v0.1/invalid/test-missing-clowder_configs.clowder.config.yml

@@ -0,0 +1,9 @@
+version: 0.1
+
+# clowder_configs: # ERROR
+#   - name: cats
+#     clowder_dir: /Users/jrgoodle/projects/cats
+#     defaults:
+#       parallel: true
+#       projects: [kit, kishka]
+

test/config/v0.1/invalid/test-missing-version.clowder.config.yml

@@ -0,0 +1,9 @@
+# version: 0.1 # ERROR
+
+clowder_configs:
+  - name: cats
+    clowder_dir: /Users/jrgoodle/projects/cats
+    defaults:
+      parallel: true
+      projects: [kit, kishka]
+