Merge pull request #573 from JrGoodle/protocol

Add protocol option to clowder herd

.circleci/config.yml

@@ -45,6 +45,7 @@ jobs:
             git config --global user.email "circle@circleci.org"
             git config --global user.name "CircleCI"
             git config --global push.default simple
+            git config --global --unset url.ssh://git@github.com.insteadOf
 
       - run:
           name: run write tests

clowder_test/clowder_test/cli/misc_controller.py

@@ -42,6 +42,14 @@ def forks(self) -> None:
 
         self._execute_command('./forks.sh', self.path)
 
+    @expose(
+        help='Run misc protocol tests'
+    )
+    def protocol(self) -> None:
+        """clowder misc protocol tests"""
+
+        self._execute_command('./protocol.sh', self.path)
+
     @expose(
         help='Run misc sources tests'
     )

examples/cats/clean.sh

@@ -17,3 +17,5 @@ rm -rf sasha
 rm -rf jrgoodle
 rm -f clowder.yaml
 rm -f clowder.yml
+rm -f $HOME/.config/clowder/clowder.config.yml
+rm -f $HOME/.config/clowder/clowder.config.yml.backup

examples/cats/init.sh

@@ -7,6 +7,9 @@ if [ -z "$COMMAND" ]; then
 fi
 
 declare -f begin_command > /dev/null && begin_command
-$COMMAND init https://github.com/jrgoodle/cats.git || exit 1
+if [ -n "$CIRCLECI" ]; then
+    $COMMAND init git@github.com:jrgoodle/cats.git || exit 1
+else
+    $COMMAND init https://github.com/jrgoodle/cats.git || exit 1
+fi
 declare -f end_command > /dev/null && end_command
-exit # Don't propagate error

examples/misc/clean.sh

@@ -12,3 +12,5 @@ rm -rf djinni
 rm -rf gyp
 rm -rf sox
 rm -rf sox-code
+rm -f $HOME/.config/clowder/clowder.config.yml
+rm -f $HOME/.config/clowder/clowder.config.yml.backup

examples/misc/init.sh

@@ -7,6 +7,9 @@ if [ -z "$COMMAND" ]; then
 fi
 
 declare -f begin_command > /dev/null && begin_command
-$COMMAND init https://github.com/jrgoodle/misc-clowder-tests.git || exit 1
+if [ -n "$CIRCLECI" ]; then
+    $COMMAND init git@github.com:jrgoodle/misc-clowder-tests.git || exit 1
+else
+    $COMMAND init https://github.com/jrgoodle/misc-clowder-tests.git || exit 1
+fi
 declare -f end_command > /dev/null && end_command
-exit # Don't propagate error

examples/swift-projects/clean.sh

@@ -35,3 +35,6 @@ export files=( 'swift' \
 for file in "${files[@]}"; do
     rm -rf "$file"
 done
+
+rm -f $HOME/.config/clowder/clowder.config.yml
+rm -f $HOME/.config/clowder/clowder.config.yml.backup

examples/swift-projects/init.sh

@@ -7,6 +7,9 @@ if [ -z "$COMMAND" ]; then
 fi
 
 declare -f begin_command > /dev/null && begin_command
-$COMMAND init https://github.com/jrgoodle/swift-clowder.git || exit 1
+if [ -n "$CIRCLECI" ]; then
+    $COMMAND init git@github.com:jrgoodle/swift-clowder.git || exit 1
+else
+    $COMMAND init https://github.com/jrgoodle/swift-clowder.git || exit 1
+fi
 declare -f end_command > /dev/null && end_command
-exit # Don't propagate error

src/clowder/cli/config.py

@@ -238,7 +238,7 @@ def config_set_protocol(args) -> None:
 
     print(' - Set protocol config value')
     config = _config()
-    config.current_clowder_config.protocol = args.protocol
+    config.current_clowder_config.protocol = args.protocol[0]
     config.save()
     print()
     config.current_clowder_config.print_configuration()

src/clowder/cli/herd.py

@@ -39,7 +39,9 @@ def add_herd_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
         (['--jobs', '-j'], dict(metavar='<n>', nargs=1, default=None, type=int,
                                 help='number of jobs to use runnning commands in parallel')),
         (['--rebase', '-r'], dict(action='store_true', help='use rebase instead of pull')),
-        (['--depth', '-d'], dict(default=None, type=int, nargs=1, metavar='<n>', help='depth to herd'))
+        (['--depth', '-d'], dict(default=None, type=int, nargs=1, metavar='<n>', help='depth to herd')),
+        (['--protocol', '-p'], dict(default=None, nargs=1, metavar='<protocol>', choices=('ssh', 'https'),
+                                    help='git protocol to use for cloning'))
     ]
 
     parser = subparsers.add_parser('herd', help='Clone and update projects with latest changes')
@@ -66,13 +68,19 @@ def herd(args) -> None:
     branch = None if args.branch is None else args.branch[0]
     tag = None if args.tag is None else args.tag[0]
     depth = None if args.depth is None else args.depth[0]
+    protocol = None if args.protocol is None else args.protocol[0]
     rebase = args.rebase
 
     config = Config(CLOWDER_CONTROLLER.name, CLOWDER_CONTROLLER.project_choices)
 
     rebase_config = config.current_clowder_config.rebase
     rebase = rebase_config if rebase_config is not None else rebase
 
+    protocol_config = config.current_clowder_config.protocol
+    protocol = protocol_config if protocol_config is not None else protocol
+    for s in CLOWDER_CONTROLLER.sources:
+        s.update_protocol(protocol)
+
     jobs = None
     if args.jobs:
         jobs = args.jobs[0]

src/clowder/model/source.py

@@ -5,6 +5,8 @@
 
 """
 
+from typing import Optional
+
 from clowder.git import GitProtocol
 
 from .defaults import Defaults
@@ -59,3 +61,12 @@ def __init__(self, source: dict, defaults: Defaults):
         super().__init__(source)
 
         self.protocol = GitProtocol(source.get('protocol', defaults.protocol))
+
+    def update_protocol(self, protocol: Optional[str]):
+        """Updates git protocol if it wasn't explicitly set for this source in the clowder yaml file
+
+        :param Optional[str] protocol: Git protocol to use for cloning
+        """
+
+        if protocol is not None and self._protocol is None:
+            self.protocol = GitProtocol(protocol)

test/scripts/cats/write_repo.sh

@@ -11,6 +11,9 @@ if [ "$ACCESS_LEVEL" == "write" ]; then
     cd "$CATS_EXAMPLE_DIR" || exit 1
     ./clean.sh
     ./init.sh || exit 1
+    begin_command
+    $COMMAND link ssh || exit 1
+    end_command
 
     begin_command
     $COMMAND repo checkout repo-test || exit 1

test/scripts/misc/forks.sh

@@ -6,14 +6,6 @@ cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." || exit 1
 
 . test_utilities.sh
 
-export project_paths=( 'djinni' \
-                       'gyp' \
-                       'sox' )
-
-export projects=( 'dropbox/djinni' \
-                  'gyp' \
-                  'p/sox/code' )
-
 print_double_separator
 echo "TEST: Test clowder forks"
 cd "$MISC_EXAMPLE_DIR" || exit 1
@@ -64,14 +56,14 @@ test_forks_env() {
     begin_command
     $COMMAND herd $PARALLEL || exit 1
     end_command
-    # echo "TEST: Fork remote environment variable in script"
+    echo "TEST: Environment variables in script"
     begin_command
     $COMMAND forall "gyp" -c "$TEST_SCRIPT_DIR/test_forall_script_env_fork.sh" $PARALLEL || exit 1
     end_command
     begin_command
     $COMMAND forall "dropbox/djinni" -c "$TEST_SCRIPT_DIR/test_forall_script_env_fork.sh" $PARALLEL && exit 1
     end_command
-    # echo "TEST: Fork remote environment variable in command"
+    echo "TEST: Environment variables in command"
     begin_command
     $COMMAND forall 'gyp' -c 'if [ $PROJECT_REMOTE != upstream ]; then exit 1; fi' $PARALLEL || exit 1
     end_command
@@ -81,10 +73,10 @@ test_forks_env() {
 }
 test_forks_env
 
-./clean.sh
-./init.sh || exit 1
-
 test_fork_herd() {
+    echo "TEST: Herd fork"
+    ./clean.sh
+    ./init.sh || exit 1
     begin_command
     $COMMAND herd $PARALLEL || exit 1
     end_command

test/scripts/misc/protocol.sh

@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+
+# set -xv
+
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." || exit 1
+
+. test_utilities.sh
+
+print_double_separator
+echo "TEST: Test clowder protocol"
+print_double_separator
+cd "$MISC_EXAMPLE_DIR" || exit 1
+
+test_herd_protocol_default_ssh() {
+    echo "TEST: Herd protocol default ssh"
+    ./clean.sh
+    ./init.sh || exit 1
+    begin_command
+    $COMMAND herd $PARALLEL || exit 1
+    end_command
+    pushd djinni || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/djinni.git'
+    test_remote_url 'upstream' 'git@github.com:dropbox/djinni.git'
+    popd || exit 1
+    pushd gyp || exit 1
+    test_branch 'fork-branch'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/gyp.git'
+    test_remote_url 'upstream' 'https://chromium.googlesource.com/external/gyp.git'
+    popd || exit 1
+    pushd sox || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/sox.git'
+    test_remote_url 'upstream' 'https://git.code.sf.net/p/sox/code.git'
+    popd || exit 1
+}
+test_herd_protocol_default_ssh
+
+test_herd_protocol_default_https() {
+    echo "TEST: Herd protocol default https"
+    ./clean.sh
+    ./init.sh || exit 1
+    begin_command
+    $COMMAND link https || exit 1
+    end_command
+    begin_command
+    $COMMAND herd $PARALLEL || exit 1
+    end_command
+    pushd djinni || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/djinni.git'
+    test_remote_url 'upstream' 'https://github.com/dropbox/djinni.git'
+    popd || exit 1
+    pushd gyp || exit 1
+    test_branch 'fork-branch'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/gyp.git'
+    test_remote_url 'upstream' 'https://chromium.googlesource.com/external/gyp.git'
+    popd || exit 1
+    pushd sox || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/sox.git'
+    test_remote_url 'upstream' 'https://git.code.sf.net/p/sox/code.git'
+    popd || exit 1
+}
+test_herd_protocol_default_https
+
+test_herd_protocol_default_ssh_override_https() {
+    echo "TEST: Herd protocol default ssh override https"
+    ./clean.sh
+    ./init.sh || exit 1
+    begin_command
+    $COMMAND herd $PARALLEL -p https || exit 1
+    end_command
+    pushd djinni || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/djinni.git'
+    test_remote_url 'upstream' 'https://github.com/dropbox/djinni.git'
+    popd || exit 1
+    pushd gyp || exit 1
+    test_branch 'fork-branch'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/gyp.git'
+    test_remote_url 'upstream' 'https://chromium.googlesource.com/external/gyp.git'
+    popd || exit 1
+    pushd sox || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/sox.git'
+    test_remote_url 'upstream' 'https://git.code.sf.net/p/sox/code.git'
+    popd || exit 1
+}
+test_herd_protocol_default_ssh_override_https
+
+test_herd_protocol_default_https_override_ssh() {
+    echo "TEST: Herd protocol default https override ssh"
+    ./clean.sh
+    ./init.sh || exit 1
+    begin_command
+    $COMMAND link https || exit 1
+    end_command
+    begin_command
+    $COMMAND herd $PARALLEL -p ssh || exit 1
+    end_command
+    pushd djinni || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/djinni.git'
+    test_remote_url 'upstream' 'git@github.com:dropbox/djinni.git'
+    popd || exit 1
+    pushd gyp || exit 1
+    test_branch 'fork-branch'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/gyp.git'
+    test_remote_url 'upstream' 'https://chromium.googlesource.com/external/gyp.git'
+    popd || exit 1
+    pushd sox || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/sox.git'
+    test_remote_url 'upstream' 'https://git.code.sf.net/p/sox/code.git'
+    popd || exit 1
+}
+test_herd_protocol_default_https_override_ssh
+
+test_herd_protocol_default_ssh_config_https() {
+    echo "TEST: Herd protocol default ssh config https"
+    ./clean.sh
+    ./init.sh || exit 1
+    begin_command
+    $COMMAND config set protocol https || exit 1
+    end_command
+    begin_command
+    $COMMAND herd $PARALLEL || exit 1
+    end_command
+    pushd djinni || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/djinni.git'
+    test_remote_url 'upstream' 'https://github.com/dropbox/djinni.git'
+    popd || exit 1
+    pushd gyp || exit 1
+    test_branch 'fork-branch'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/gyp.git'
+    test_remote_url 'upstream' 'https://chromium.googlesource.com/external/gyp.git'
+    popd || exit 1
+    pushd sox || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'https://github.com/JrGoodle/sox.git'
+    test_remote_url 'upstream' 'https://git.code.sf.net/p/sox/code.git'
+    popd || exit 1
+}
+test_herd_protocol_default_ssh_config_https
+
+test_herd_protocol_default_https_config_ssh() {
+    echo "TEST: Herd protocol default https config ssh"
+    ./clean.sh
+    ./init.sh || exit 1
+    begin_command
+    $COMMAND link https || exit 1
+    end_command
+    begin_command
+    $COMMAND config set protocol ssh || exit 1
+    end_command
+    begin_command
+    $COMMAND herd $PARALLEL || exit 1
+    end_command
+    pushd djinni || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/djinni.git'
+    test_remote_url 'upstream' 'git@github.com:dropbox/djinni.git'
+    popd || exit 1
+    pushd gyp || exit 1
+    test_branch 'fork-branch'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/gyp.git'
+    test_remote_url 'upstream' 'https://chromium.googlesource.com/external/gyp.git'
+    popd || exit 1
+    pushd sox || exit 1
+    test_branch 'master'
+    test_remote_url 'origin' 'git@github.com:JrGoodle/sox.git'
+    test_remote_url 'upstream' 'https://git.code.sf.net/p/sox/code.git'
+    popd || exit 1
+}
+test_herd_protocol_default_https_config_ssh

test/scripts/misc/sources.sh

@@ -6,14 +6,6 @@ cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.." || exit 1
 
 . test_utilities.sh
 
-export project_paths=( 'djinni' \
-                       'gyp' \
-                       'sox' )
-
-export projects=( 'dropbox/djinni' \
-                  'external/gyp' \
-                  'p/sox/code' )
-
 print_double_separator
 echo "TEST: Test clowder sources"
 cd "$MISC_EXAMPLE_DIR" || exit 1

test/scripts/test_example_misc.sh

@@ -12,3 +12,4 @@ print_double_separator
 
 "$TEST_SCRIPT_DIR/misc/sources.sh" || exit 1
 "$TEST_SCRIPT_DIR/misc/forks.sh" || exit 1
+"$TEST_SCRIPT_DIR/misc/protocol.sh" || exit 1