Merge pull request #524 from JrGoodle/config

Add clowder config command

.github/workflows/ci-cats-config.yml

@@ -0,0 +1,55 @@
+name: 'CI - cats config'
+
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  cats_config:
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+      OS_NAME: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest] # TODO: windows-latest
+        python-version: ['3.6', '3.7', '3.8']
+        exclude:
+          - os: macos-latest
+            python-version: '3.6'
+          - os: macos-latest
+            python-version: '3.8'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Setup git config
+      run:
+        git config --global user.email "joe@polka.cat";
+        git config --global user.name "GitHub Actions Workflow"
+    - name: Install clowder requirements
+      run: pip3 install -r src/requirements.txt
+    - name: CI before script
+      run: script/ci_before
+    - name: Install clowder
+      run: script/update
+    - name: Install clowder test
+      run: script/test
+    - name: Test config commands
+      run: clowder-test -c cats config
+    - name: CI after script
+      run: script/ci_after

.github/workflows/ci-config-yaml-validation.yml

@@ -0,0 +1,55 @@
+name: 'CI - config yaml validation'
+
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  config_yaml_validation:
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+      OS_NAME: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest] # TODO: windows-latest
+        python-version: ['3.6', '3.7'] # '3.8' # TODO: Fix json generation on Python 3.8
+        exclude:
+          - os: macos-latest
+            python-version: '3.6'
+          - os: macos-latest
+            python-version: '3.8'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Setup git config
+      run:
+        git config --global user.email "joe@polka.cat";
+        git config --global user.name "GitHub Actions Workflow"
+    - name: Install clowder requirements
+      run: pip3 install -r src/requirements.txt
+    - name: CI before script
+      run: script/ci_before
+    - name: Install clowder
+      run: script/update
+    - name: Install clowder test
+      run: script/test
+    - name: Test config yaml validation
+      run: clowder-test -c config-yaml-validation
+    - name: CI after script
+      run: script/ci_after

.gitignore

@@ -1,5 +1,8 @@
 # Example files
 
+test/config/*/invalid/*.json
+test/config/*/*.json
+
 examples/cats/cached
 examples/cats/clowder
 examples/cats/.clowder

clowder_test/clowder_test/cli/base_controller.py

@@ -56,6 +56,19 @@ def all(self) -> None:
         self.offline()
         self.parallel()
 
+    @expose(
+        help='Run config yaml validation tests'
+    )
+    def config_yaml_validation(self) -> None:
+        """clowder config yaml validation tests"""
+
+        execute_test_command('./test_config_yaml_validation.sh', self.path,
+                             parallel=True,
+                             write=self.app.pargs.write,
+                             coverage=self.app.pargs.coverage,
+                             debug=self.app.debug,
+                             quiet=self.app.pargs.silent)
+
     @expose(
         help='Run offline tests'
     )

clowder_test/clowder_test/cli/cats_controller.py

@@ -58,6 +58,14 @@ def clean(self) -> None:
 
         self._execute_command('./clean.sh', self.path)
 
+    @expose(
+        help='Run cats config tests'
+    )
+    def config(self) -> None:
+        """clowder cats config tests"""
+
+        self._execute_command('./config.sh', self.path)
+
     @expose(
         help='Run cats diff tests'
     )

clowder_test/setup.py

@@ -29,5 +29,5 @@
             'clowder-test=clowder_test.clowder_test_app:main',
         ]
     },
-    install_requires=['cement', 'colorama', 'coverage', 'cprint', 'psutil', 'termcolor']
+    install_requires=['cement', 'colorama', 'coverage', 'cprint', 'psutil', 'remarshal', 'termcolor']
 )

script/clean

@@ -37,6 +37,8 @@ clean_clowder_test() {
     pushd $REPO_PATH || exit 1
     sudo -H pip3 uninstall clowder_test
     popd || exit 1
+    sudo -H rm -rf $REPO_PATH/test/config/*/invalid/*.json
+    sudo -H rm -rf $REPO_PATH/test/config/*/*.json
 }
 
 clean_examples() {

src/clowder/__init__.py

@@ -6,14 +6,14 @@
 """
 
 import os
-import pkg_resources
+from pathlib import Path
 from typing import Optional
 
 
-CLOWDER_SCHEMA = pkg_resources.resource_string(__name__, "clowder.schema.json")
-
-
+PRINT_DEBUG_OUTPUT = "CLOWDER_DEBUG" in os.environ
 CURRENT_DIR = os.getcwd()
+CLOWDER_CONFIG_DIR = str(Path.home()/'.config'/'clowder')
+CLOWDER_CONFIG_YAML = str(Path(CLOWDER_CONFIG_DIR)/'clowder.config.yaml')
 CLOWDER_DIR: Optional[str] = None
 CLOWDER_REPO_DIR: Optional[str] = None
 CLOWDER_REPO_VERSIONS_DIR: Optional[str] = None
@@ -33,6 +33,7 @@
     else:
         temp_dir = os.path.dirname(temp_dir)
 
+
 # If clowder repo exists, try to set other global path variables
 if CLOWDER_REPO_DIR is not None:
     clowder_yaml = os.path.join(CLOWDER_DIR, 'clowder.yaml')

src/clowder/cli/__init__.py

@@ -8,6 +8,7 @@
 from clowder.cli.branch import add_branch_parser
 from clowder.cli.checkout import add_checkout_parser
 from clowder.cli.clean import add_clean_parser
+from clowder.cli.config import add_config_parser
 from clowder.cli.diff import add_diff_parser
 from clowder.cli.forall import add_forall_parser
 from clowder.cli.herd import add_herd_parser

src/clowder/cli/branch.py

@@ -7,13 +7,15 @@
 
 import argparse
 
+import clowder.util.formatting as fmt
 from clowder.clowder_controller import CLOWDER_CONTROLLER
+from clowder.config import Config
 from clowder.util.clowder_utils import (
     add_parser_arguments,
-    filter_projects,
-    options_help_message
+    filter_projects
 )
 from clowder.util.decorators import (
+    print_clowder_name,
     print_clowder_repo_status,
     valid_clowder_yaml_required
 )
@@ -22,9 +24,10 @@
 def add_branch_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
 
     arguments = [
-        (['projects'], dict(metavar='PROJECT', default='all', nargs='*', choices=CLOWDER_CONTROLLER.project_choices,
-                            help=options_help_message(CLOWDER_CONTROLLER.project_names,
-                                                      'projects and groups to show branches for')))
+        (['projects'], dict(metavar='PROJECT', default='default', nargs='*',
+                            choices=CLOWDER_CONTROLLER.project_choices_with_default,
+                            help=fmt.options_help_message(CLOWDER_CONTROLLER.project_choices,
+                                                          'projects and groups to show branches for')))
     ]
 
     parser = subparsers.add_parser('branch', help='Display current branches')
@@ -40,14 +43,10 @@ def add_branch_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
     parser.set_defaults(func=branch)
 
 
-def branch(args) -> None:
-    """Clowder branch command entry point"""
-    _branch(args)
-
-
 @valid_clowder_yaml_required
+@print_clowder_name
 @print_clowder_repo_status
-def _branch(args) -> None:
+def branch(args) -> None:
     """Clowder branch command private implementation"""
     if args.remote:
         local = False
@@ -59,7 +58,10 @@ def _branch(args) -> None:
         local = True
         remote = False
 
-    projects = filter_projects(CLOWDER_CONTROLLER.projects, args.projects)
+    config = Config(CLOWDER_CONTROLLER.name, CLOWDER_CONTROLLER.project_choices)
+    projects = config.process_projects_arg(args.projects)
+    projects = filter_projects(CLOWDER_CONTROLLER.projects, projects)
+
     for project in projects:
         print(project.status())
         project.branch(local=local, remote=remote)

src/clowder/cli/checkout.py

@@ -7,13 +7,15 @@
 
 import argparse
 
+import clowder.util.formatting as fmt
 from clowder.clowder_controller import CLOWDER_CONTROLLER
+from clowder.config import Config
 from clowder.util.clowder_utils import (
     add_parser_arguments,
-    filter_projects,
-    options_help_message
+    filter_projects
 )
 from clowder.util.decorators import (
+    print_clowder_name,
     print_clowder_repo_status,
     valid_clowder_yaml_required
 )
@@ -23,29 +25,27 @@ def add_checkout_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
 
     arguments = [
         (['branch'], dict(nargs=1, action='store', help='branch to checkout', metavar='BRANCH')),
-        (['projects'], dict(metavar='PROJECT', default='all', nargs='*',
-                            choices=CLOWDER_CONTROLLER.project_choices,
-                            help=options_help_message(CLOWDER_CONTROLLER.project_names,
-                                                      'projects and groups to checkout branches for')))
+        (['projects'], dict(metavar='PROJECT', default='default', nargs='*',
+                            choices=CLOWDER_CONTROLLER.project_choices_with_default,
+                            help=fmt.options_help_message(CLOWDER_CONTROLLER.project_choices,
+                                                          'projects and groups to checkout branches for')))
     ]
 
     parser = subparsers.add_parser('checkout', help='Checkout local branch in projects')
     add_parser_arguments(parser, arguments)
     parser.set_defaults(func=checkout)
 
 
-def checkout(args) -> None:
-    """Clowder checkout command entry point"""
-
-    _checkout(args)
-
-
 @valid_clowder_yaml_required
+@print_clowder_name
 @print_clowder_repo_status
-def _checkout(args) -> None:
+def checkout(args) -> None:
     """Clowder checkout command private implementation"""
 
-    projects = filter_projects(CLOWDER_CONTROLLER.projects, args.projects)
+    config = Config(CLOWDER_CONTROLLER.name, CLOWDER_CONTROLLER.project_choices)
+    projects = config.process_projects_arg(args.projects)
+    projects = filter_projects(CLOWDER_CONTROLLER.projects, projects)
+
     for project in projects:
         print(project.status())
         project.checkout(args.branch[0])