Merge pull request #533 from JrGoodle/yml

Add support for .yml extension

.gitignore

@@ -7,7 +7,7 @@ examples/cats/cached
 examples/cats/clowder
 examples/cats/.clowder
 examples/cats/clowder.yaml
-examples/cats/peru.yaml
+examples/cats/clowder.yml
 examples/cats/black*
 examples/cats/mu
 examples/cats/mu-cat
@@ -23,8 +23,8 @@ examples/cats/jrgoodle
 examples/swift-projects/cached
 examples/swift-projects/clowder
 examples/swift-projects/.clowder
+examples/swift-projects/clowder.yml
 examples/swift-projects/clowder.yaml
-examples/swift-projects/peru.yaml
 examples/swift-projects/clang
 examples/swift-projects/cmark
 examples/swift-projects/compiler-rt
@@ -43,6 +43,7 @@ examples/swift-projects/swiftpm
 examples/misc/cached
 examples/misc/clowder
 examples/misc/.clowder
+examples/misc/clowder.yml
 examples/misc/clowder.yaml
 examples/misc/djinni
 examples/misc/gyp

README.md

@@ -25,7 +25,7 @@ Managing multiple repositories can be pretty frustrating. There are a number of
 
 All of these have their own approach, but many are based on submodules or subtrees. Submodules and subtrees create a tight coupling between repositories because of the way dependencies are stored. Much has been written about their drawbacks elsewhere. Google's `repo` tool takes a different approach, but is closely tied to Google's development workflow
 
-`clowder` uses a similar approach as `repo`, but using yaml instead of xml for the configuration file. URL information and relative project locations on disk are specified in a `clowder.yaml` file. This file is checked into its own repository. The use of a separate file to track projects allows for detailed information about the dependencies between them to be stored, but each repository is still essentially independent. Projects can track branches, or be tied to specific tags or commits
+`clowder` uses a similar approach as `repo`, but using yaml instead of xml for the configuration file. URL information and relative project locations on disk are specified in a `clowder.yml` or `clowder.yaml` file. This file is checked into its own repository. The use of a separate file to track projects allows for detailed information about the dependencies between them to be stored, but each repository is still essentially independent. Projects can track branches, or be tied to specific tags or commits
 
 The primary purpose of `clowder` is synchronization of multiple repositories, so normal development still takes place in individual repositories with the usual `git` commands
 

examples/cats/clean.sh

@@ -16,3 +16,4 @@ rm -rf june
 rm -rf sasha
 rm -rf jrgoodle
 rm -f clowder.yaml
+rm -f clowder.yml

examples/misc/clean.sh

@@ -7,6 +7,7 @@ echo 'Clean misc example directory'
 rm -rf clowder
 rm -rf .clowder
 rm -f clowder.yaml
+rm -f clowder.yml
 rm -rf djinni
 rm -rf gyp
 rm -rf sox

examples/swift-projects/clean.sh

@@ -21,3 +21,4 @@ rm -rf swift-integration-tests
 rm -rf swift-xcode-playground-support
 rm -rf swiftpm
 rm -f clowder.yaml
+rm -f clowder.yml

src/clowder/__init__.py

@@ -5,24 +5,42 @@
 
 """
 
-# import logging
+import logging
 import os
+import traceback
 from pathlib import Path
 from typing import Optional
 
+from clowder.error import ClowderExit
+from clowder.util.formatting import ERROR
+
+
+# Set up logging #
 
 PRINT_DEBUG_OUTPUT = "CLOWDER_DEBUG" in os.environ
-# logging.basicConfig()
-# logging.raiseExceptions = True
-# if PRINT_DEBUG_OUTPUT:
-#     logging.getLogger("CLOWDER DEBUG").setLevel(logging.DEBUG)
-# else:
-#     logging.getLogger("CLOWDER DEBUG").setLevel(logging.ERROR)
-# LOG_DEBUG = logging.getLogger("CLOWDER DEBUG")
+logging.basicConfig()
+logging.raiseExceptions = True
+logger = logging.getLogger("CLOWDER")
+if PRINT_DEBUG_OUTPUT:
+    logger.setLevel(logging.DEBUG)
+else:
+    logger.setLevel(logging.ERROR)
+
+
+def LOG_DEBUG(message: str, exception: Optional[Exception] = None): # noqa
+    if PRINT_DEBUG_OUTPUT:
+        logger.log(logging.DEBUG, f" {message}")
+        if exception is not None:
+            logger.log(logging.DEBUG, str(exception))
+            # TODO: Format the output for clowder debug
+            traceback.print_exc()
+
+
+# Set up global paths #
 
 CURRENT_DIR = Path.cwd()
 CLOWDER_CONFIG_DIR = Path.home() / '.config' / 'clowder'
-CLOWDER_CONFIG_YAML = CLOWDER_CONFIG_DIR / 'clowder.config.yaml'
+CLOWDER_CONFIG_YAML = CLOWDER_CONFIG_DIR / 'clowder.config.yml'
 CLOWDER_DIR: Optional[Path] = None
 CLOWDER_REPO_DIR: Optional[Path] = None
 CLOWDER_REPO_VERSIONS_DIR: Optional[Path] = None
@@ -43,8 +61,18 @@
 
 # If clowder repo exists, try to set other global path variables
 if CLOWDER_REPO_DIR is not None:
+    clowder_yml = CLOWDER_DIR / 'clowder.yml'
     clowder_yaml = CLOWDER_DIR / 'clowder.yaml'
-    if clowder_yaml.is_symlink():
+    if clowder_yml.is_symlink() and clowder_yaml.is_symlink():
+        print(f"\n{ERROR} Found clowder.yml and clowder.yaml files in same directory\n")
+        try:
+            raise ClowderExit(1)
+        except ClowderExit as err:
+            LOG_DEBUG('Ambigiuous clowder file', err)
+            exit(err.code)
+    if clowder_yml.is_symlink():
+        CLOWDER_YAML = clowder_yml
+    elif clowder_yaml.is_symlink():
         CLOWDER_YAML = clowder_yaml
 
     clowder_versions = CLOWDER_REPO_DIR / 'versions'

src/clowder/cli/init.py

@@ -20,11 +20,11 @@
 def add_init_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
 
     arguments = [
-        (['url'], dict(metavar='URL', help='url of repo containing clowder.yaml')),
-        (['--branch', '-b'], dict(nargs=1, metavar='BRANCH', help='branch of repo containing clowder.yaml'))
+        (['url'], dict(metavar='URL', help='url of repo containing clowder yaml file')),
+        (['--branch', '-b'], dict(nargs=1, metavar='BRANCH', help='branch of repo containing clowder yaml file'))
     ]
 
-    parser = subparsers.add_parser('init', help='Clone repository to clowder directory and create clowder.yaml symlink')
+    parser = subparsers.add_parser('init', help='Clone repository to clowder directory and create clowder yaml symlink')
     add_parser_arguments(parser, arguments)
     parser.set_defaults(func=init)
 

src/clowder/cli/link.py

@@ -12,7 +12,8 @@
 from clowder.util.clowder_utils import (
     add_parser_arguments,
     get_saved_version_names,
-    link_clowder_yaml
+    link_clowder_yaml_default,
+    link_clowder_yaml_version
 )
 from clowder.util.decorators import (
     print_clowder_name,
@@ -30,7 +31,7 @@ def add_link_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
                            help=fmt.options_help_message(versions, 'version to symlink')))
     ]
 
-    parser = subparsers.add_parser('link', help='Symlink clowder.yaml version')
+    parser = subparsers.add_parser('link', help='Symlink clowder yaml version')
     add_parser_arguments(parser, arguments)
     parser.set_defaults(func=link)
 
@@ -41,4 +42,7 @@ def add_link_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
 def link(args) -> None:
     """Clowder link command private implementation"""
 
-    link_clowder_yaml(CLOWDER_DIR, args.version)
+    if args.version is None:
+        link_clowder_yaml_default(CLOWDER_DIR)
+    else:
+        link_clowder_yaml_version(CLOWDER_DIR, args.version)

src/clowder/cli/save.py

@@ -30,7 +30,7 @@ def add_save_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
         (['version'], dict(help='version to save', metavar='VERSION'))
     ]
 
-    parser = subparsers.add_parser('save', help='Create version of clowder.yaml for current repos')
+    parser = subparsers.add_parser('save', help='Create clowder yaml version for current repos')
     add_parser_arguments(parser, arguments)
     parser.set_defaults(func=save)
 
@@ -59,10 +59,14 @@ def save(args) -> None:
     versions_dir = CLOWDER_REPO_DIR / 'versions'
     make_dir(versions_dir)
 
+    yml_file = versions_dir / f"{version_name}.clowder.yml"
     yaml_file = versions_dir / f"{version_name}.clowder.yaml"
-    if yaml_file.exists():
+    if yml_file.exists():
+        print(f"{fmt.error_save_version_exists(version_name, yml_file)}\n")
+        raise ClowderExit(1)
+    elif yaml_file.exists():
         print(f"{fmt.error_save_version_exists(version_name, yaml_file)}\n")
         raise ClowderExit(1)
 
-    print(fmt.save_version_message(version_name, yaml_file))
-    save_yaml(CLOWDER_CONTROLLER.get_yaml(), yaml_file)
+    print(fmt.save_version_message(version_name, yml_file))
+    save_yaml(CLOWDER_CONTROLLER.get_yaml(), yml_file)

src/clowder/cli/yaml.py

@@ -21,10 +21,10 @@
 def add_yaml_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
 
     arguments = [
-        (['--resolved', '-r'], dict(action='store_true', help='print resolved clowder.yaml'))
+        (['--resolved', '-r'], dict(action='store_true', help='print resolved clowder yaml file'))
     ]
 
-    parser = subparsers.add_parser('yaml', help='Print clowder.yaml information')
+    parser = subparsers.add_parser('yaml', help='Print clowder yaml file information')
     add_parser_arguments(parser, arguments)
     parser.set_defaults(func=yaml)
 

src/clowder/clowder_app.py

@@ -13,7 +13,7 @@
 import colorama
 
 import clowder.cli as cmd
-from clowder import PRINT_DEBUG_OUTPUT
+from clowder import LOG_DEBUG
 from clowder.error import ClowderExit
 from clowder.util.clowder_utils import add_parser_arguments
 from clowder.util.parallel import __clowder_pool__
@@ -71,31 +71,26 @@ def main() -> None:
     """Clowder command CLI main function"""
 
     print()
+    exit_code = 0
+
     parser = create_parsers()
     argcomplete.autocomplete(parser)
     args = parser.parse_args()
     if 'projects' in args:
         if isinstance(args.projects, str):
             args.projects = [args.projects]
-
-    exit_code = 0
     try:
         args.func(args) # noqa
     except ClowderExit as err:
-        if PRINT_DEBUG_OUTPUT:
-            print('ClowderExit exception')
-            print(err)
+        LOG_DEBUG('ClowderExit exception', err)
         exit_code = err.code
     except AttributeError as err:
-        if PRINT_DEBUG_OUTPUT:
-            print('AttributeError exception')
-            print(err)
+        LOG_DEBUG('AttributeError exception', err)
         exit_code = 1
-        parser.print_help()
+        if parser is not None:
+            parser.print_help()
     except Exception as err: # noqa
-        if PRINT_DEBUG_OUTPUT:
-            print('Unhandled generic exception')
-            print(err)
+        LOG_DEBUG('Unhandled generic exception', err)
         exit_code = 1
     finally:
         print()

src/clowder/clowder_controller.py

@@ -24,16 +24,16 @@
 
 
 class ClowderController(object):
-    """Class encapsulating project information from clowder.yaml for controlling clowder
+    """Class encapsulating project information from clowder yaml for controlling clowder
 
     :ivar Optional[str] name: Name of clowder
-    :ivar Optional[Defaults] defaults: Global clowder.yaml defaults
+    :ivar Optional[Defaults] defaults: Global clowder yaml defaults
     :ivar Tuple[Group, ...] groups: List of all Groups
     :ivar Tuple[Source, ...] sources: List of all Sources
     :ivar Tuple[str, ...] project_names: All possible project and group names
     :ivar Tuple[str, ...] project_choices: All possible project and group choices
     :ivar Tuple[str, ...] project_choices_with_default: All possible project and group choices, including 'default'
-    :ivar Optional[Exception] error: Exception from failing to load clowder.yaml
+    :ivar Optional[Exception] error: Exception from failing to load clowder yaml file
     """
 
     def __init__(self):
@@ -152,7 +152,7 @@ def validate_projects_exist(self) -> None:
             raise ClowderExit(1)
 
     def _get_all_project_names(self) -> Tuple[str, ...]:
-        """Returns all project names for current clowder.yaml
+        """Returns all project names for current clowder yaml file
 
         :return: All project and group names
         :rtype: Tuple
@@ -165,7 +165,7 @@ def _get_all_project_names(self) -> Tuple[str, ...]:
             return ()
 
     def _load_clowder_yaml(self, yaml: dict) -> None:
-        """Load clowder.yaml
+        """Load clowder yaml file
 
         :param dict yaml: Parsed yaml dict
         """

src/clowder/clowder_repo.py

@@ -16,7 +16,7 @@
 from clowder.error import ClowderError, ClowderExit
 from clowder.git import ProjectRepo
 from clowder.git.util import existing_git_repository
-from clowder.util.clowder_utils import link_clowder_yaml
+from clowder.util.clowder_utils import link_clowder_yaml_default
 from clowder.util.connectivity import is_offline
 from clowder.util.execute import execute_command
 from clowder.util.file_system import remove_directory
@@ -107,7 +107,7 @@ def init(url: str, branch: str) -> None:
     clowder_repo_dir = CURRENT_DIR / '.clowder'
     repo = ProjectRepo(clowder_repo_dir, clowder_repo_remote, clowder_repo_ref)
     repo.create_clowder_repo(url, branch)
-    link_clowder_yaml(CURRENT_DIR)
+    link_clowder_yaml_default(CURRENT_DIR)
 
 
 def print_status(fetch: bool = False) -> None:
@@ -132,15 +132,13 @@ def print_status(fetch: bool = False) -> None:
     current_ref_output = repo.format_project_ref_string()
 
     if CLOWDER_YAML is None or not CLOWDER_YAML.is_symlink():
-        print(project_output + ' ' + current_ref_output)
+        print(f"{project_output} {current_ref_output}")
         return
 
-    clowder_file = Path('clowder.yaml')
-    symlink_output = fmt.path_string(clowder_file)
-    clowder_path = CLOWDER_YAML.resolve().relative_to(CLOWDER_DIR)
-    path_output = fmt.path_string(Path(str(clowder_path)[1:-1]))
-    print(f'{project_output} {current_ref_output}')
-    print(f'{symlink_output} -> {path_output}\n')
+    symlink_path = fmt.path_string(Path(CLOWDER_YAML.name))
+    file_path = fmt.path_string(CLOWDER_YAML.resolve().relative_to(CLOWDER_DIR))
+    print(f"{project_output} {current_ref_output}")
+    print(f"{symlink_path} -> {file_path}\n")
 
 
 def pull() -> None:
@@ -171,14 +169,15 @@ def run_command(command: str) -> None:
 
 
 def _init_exit_handler() -> None:
-    """Exit handler for deleting files if init fails
+    """Exit handler for deleting files if clowder init fails
 
     :raise ClowderExit:
     """
 
     clowder_path = CURRENT_DIR / '.clowder'
     if os.path.isdir(clowder_path):
+        clowder_yml = CURRENT_DIR / 'clowder.yml'
         clowder_yaml = CURRENT_DIR / 'clowder.yaml'
-        if not clowder_yaml.is_symlink():
+        if not clowder_yml.is_symlink() and not clowder_yaml.is_symlink():
             remove_directory(clowder_path)
             raise ClowderExit(1)

src/clowder/config/clowder_config.py

@@ -182,7 +182,7 @@ def print_configuration(self) -> None:
 
     @staticmethod
     def validate_config_projects_defined(project_names: Tuple[str, ...], project_options: Tuple[str, ...]) -> None:
-        """Validate all projects were defined in clowder.yaml
+        """Validate all projects were defined in clowder yaml file
 
         :param Tuple[str, ...] project_names: Project names to validate
         :param Tuple[str, ...] project_options: Projects to validate against

src/clowder/config/config.py

@@ -33,7 +33,7 @@ class Config(object):
     :ivar float version: Version number of config file
     :ivar Tuple[ClowderConfig, ...] clowder_configs: Configs for clowders
     :ivar Optional[ClowderConfig] current_clowder_config: Config for current clowder
-    :ivar Optional[Exception] error: Exception from failing to load clowder.config.yaml
+    :ivar Optional[Exception] error: Exception from failing to load clowder config yaml file
     """
 
     def __init__(self, current_clowder_name: Optional[str], project_options: Tuple[str, ...]):
@@ -141,7 +141,7 @@ def _get_yaml(self) -> dict:
         return config
 
     def _load_clowder_config_yaml(self) -> None:
-        """Load clowder.config.yaml
+        """Load clowder config yaml file
 
         :raise ClowderExit:
         """

src/clowder/git/git_repo.py

@@ -32,7 +32,7 @@ class GitRepo(object):
     :ivar str default_ref: Default ref
     :ivar str remote: Default remote name
     :ivar bool parallel: Whether command is being run in parallel, affects output
-    :ivar Repo repo: Repo instance
+    :ivar Repo Optional[repo]: Repo instance
     """
 
     def __init__(self, repo_path: Path, remote: str, default_ref: str, parallel: bool = False):
@@ -230,7 +230,7 @@ def format_project_ref_string(self) -> str:
             status = f'({local_commits_output}/{upstream_commits_output})'
 
         if self.is_detached():
-            current_ref = self.sha(short=True)
+            current_ref = self.sha()
             return colored(f'[HEAD @ {current_ref}]', 'magenta')
         current_branch = self.current_branch()
         return colored(f'[{current_branch}]', 'magenta') + status