Merge pull request #469 from JrGoodle/forks

Update fork handling

.circleci/config.yml

@@ -51,7 +51,7 @@ jobs:
 
       - run:
           name: run misc tests
-          command: clowder-test -c misc
+          command: clowder-test -c misc all
 
       - run:
           name: upload code coverage results

clowder_test/clowder_test/cli/base_controller.py

@@ -9,7 +9,7 @@
 
 from cement.ext.ext_argparse import ArgparseController, expose
 
-from clowder_test.execute import execute_test_command
+from clowder_test.execute import create_misc_cache, execute_test_command
 from clowder_test import ROOT_DIR
 
 
@@ -57,19 +57,6 @@ def all(self) -> None:
         self.offline()
         self.parallel()
 
-    @expose(
-        help='Run misc tests'
-    )
-    def misc(self) -> None:
-        """clowder misc tests"""
-
-        execute_test_command('./test_example_misc.sh', self.path,
-                             parallel=self.app.pargs.parallel,
-                             write=self.app.pargs.write,
-                             coverage=self.app.pargs.coverage,
-                             debug=self.app.debug,
-                             quiet=self.app.pargs.silent)
-
     @expose(
         help='Run offline tests'
     )
@@ -113,6 +100,7 @@ def parallel(self) -> None:
     @expose(
         help='Run tests requiring remote write permissions'
     )
+    @create_misc_cache
     def write(self) -> None:
         """clowder write tests"""
 
@@ -125,14 +113,14 @@ def write(self) -> None:
                                  debug=self.app.debug,
                                  quiet=self.app.pargs.silent)
 
-        # llvm_scripts = ['./write_forks.sh', './write_sync.sh']
-        # for script in llvm_scripts:
-        #     execute_test_command(script, os.path.join(self.path, 'llvm'),
-        #                          parallel=self.app.pargs.parallel,
-        #                          write=True,
-        #                          coverage=self.app.pargs.coverage,
-        #                          debug=self.app.debug,
-        #                          quiet=self.app.pargs.silent)
+        misc_scripts = ['./write_forks.sh']
+        for script in misc_scripts:
+            execute_test_command(script, os.path.join(self.path, 'misc'),
+                                 parallel=self.app.pargs.parallel,
+                                 write=True,
+                                 coverage=self.app.pargs.coverage,
+                                 debug=self.app.debug,
+                                 quiet=self.app.pargs.silent)
 
         # execute_test_command('./write_configure_remotes.sh', os.path.join(self.path, 'swift'),
         #                      parallel=self.app.pargs.parallel,

clowder_test/clowder_test/cli/llvm_controller.py

@@ -34,14 +34,14 @@ def all(self) -> None:
 
         self._execute_command('./test_example_llvm.sh', os.path.join(ROOT_DIR, 'test', 'scripts'))
 
-    @expose(
-        help='Run llvm forks tests'
-    )
-    @create_llvm_cache
-    def forks(self) -> None:
-        """clowder llvm forks tests"""
-
-        self._execute_command('./forks.sh', self.path)
+    # @expose(
+    #     help='Run llvm forks tests'
+    # )
+    # @create_llvm_cache
+    # def forks(self) -> None:
+    #     """clowder llvm forks tests"""
+    #
+    #     self._execute_command('./forks.sh', self.path)
 
     # @expose(
     #     help='Run llvm sync tests'

clowder_test/clowder_test/cli/misc_controller.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+"""Clowder test misc command line controller
+
+.. codeauthor:: Joe Decapo <joe@polka.cat>
+
+"""
+
+import os
+
+from cement.ext.ext_argparse import ArgparseController, expose
+
+from clowder_test.execute import create_misc_cache, execute_test_command
+from clowder_test import ROOT_DIR
+
+
+class MiscController(ArgparseController):
+    """Clowder test command misc controller"""
+
+    path = os.path.join(ROOT_DIR, 'test', 'scripts', 'misc')
+
+    class Meta:
+        """Clowder test misc Meta configuration"""
+
+        label = 'misc'
+        stacked_on = 'base'
+        stacked_type = 'nested'
+        description = 'Run misc tests'
+
+    @expose(
+        help='Run all misc tests'
+    )
+    def all(self) -> None:
+        """clowder misc tests"""
+
+        self._execute_command('./test_example_misc.sh', os.path.join(ROOT_DIR, 'test', 'scripts'))
+
+    @expose(
+        help='Run misc forks tests'
+    )
+    @create_misc_cache
+    def forks(self) -> None:
+        """clowder misc forks tests"""
+
+        self._execute_command('./forks.sh', self.path)
+
+    @expose(
+        help='Run misc sources tests'
+    )
+    @create_misc_cache
+    def sources(self) -> None:
+        """clowder misc sources tests"""
+
+        self._execute_command('./sources.sh', self.path)
+
+    def _execute_command(self, command: str, path: str) -> None:
+        """Private execute command
+
+        :param str command: Command to run
+        :param str path: Path to set as ``cwd``
+        """
+
+        execute_test_command(command, path,
+                             parallel=self.app.pargs.parallel,
+                             write=self.app.pargs.write,
+                             coverage=self.app.pargs.coverage,
+                             debug=self.app.debug,
+                             quiet=self.app.pargs.silent)

clowder_test/clowder_test/clowder_test_app.py

@@ -14,11 +14,12 @@
 from clowder_test.cli.base_controller import BaseController
 from clowder_test.cli.cats_controller import CatsController
 from clowder_test.cli.llvm_controller import LLVMController
+from clowder_test.cli.misc_controller import MiscController
 from clowder_test.cli.swift_controller import SwiftController
 from clowder_test.execute import execute_command
 
 
-def post_argument_parsing_hook(app):
+def post_argument_parsing_hook(app): # noqa
     execute_command('./setup_local_test_directory.sh', os.path.join(ROOT_DIR, 'test', 'scripts'))
 
 
@@ -37,6 +38,7 @@ class Meta:
             BaseController,
             CatsController,
             LLVMController,
+            MiscController,
             SwiftController
         ]
 

src/clowder/model/fork.py

@@ -25,7 +25,8 @@ class Fork(object):
 
     :ivar str name: Project name
     :ivar str path: Project relative path
-    :ivar str remote: Git remote name
+    :ivar str remote: Fork remote name
+    :ivar str ref: Fork git ref
     """
 
     def __init__(self, fork: dict, path: str, project_name: str, project_source: Source,
@@ -45,7 +46,7 @@ def __init__(self, fork: dict, path: str, project_name: str, project_source: Sou
         self.path = path
         self.name = fork['name']
         self.remote = fork.get('remote', defaults.remote)
-        self._ref = fork.get('ref', project_ref)
+        self.ref = fork.get('ref', project_ref)
         self._recursive = recursive
 
         self._source = None
@@ -92,7 +93,7 @@ def status(self) -> str:
         if not existing_git_repository(self.path):
             return colored(self.path, 'green')
 
-        repo = ProjectRepo(self.full_path(), self.remote, self._ref)
+        repo = ProjectRepo(self.full_path(), self.remote, self.ref)
         project_output = repo.format_project_string(self.path)
         current_ref_output = repo.format_project_ref_string()
         return project_output + ' ' + current_ref_output

src/clowder/model/project.py

@@ -50,8 +50,8 @@ class Project(object):
     :ivar Optional[str] alias: Alias for project
     :ivar str path: Project relative path
     :ivar List[str] groups: Groups project belongs to
-    :ivar str ref: Default git ref
-    :ivar str remote: Default remote name
+    :ivar str ref: Project git ref
+    :ivar str remote: Project remote name
     :ivar int depth: Depth to clone project repo
     :ivar bool recursive: Whether to recursively clone submodules
     :ivar Source source: Default source
@@ -240,8 +240,11 @@ def get_yaml(self, resolved: bool = False) -> dict:
         if resolved:
             ref = self.ref
         else:
-            repo = ProjectRepo(self.full_path(), self.remote, self.ref)
-            ref = repo.sha()
+            if self.fork is None:
+                ref = self.ref
+            else:
+                repo = ProjectRepo(self.full_path(), self.remote, self.ref)
+                ref = repo.sha()
 
         project = {'name': self.name,
                    'path': self.path,
@@ -279,34 +282,35 @@ def herd(self, branch: Optional[str] = None, tag: Optional[str] = None, depth: O
 
         herd_depth = self.depth if depth is None else depth
         repo = self._repo(self.recursive, parallel=parallel)
-        fork_remote = None if self.fork is None else self.fork.remote
-
-        if branch:
-            command = 'herd_branch'
-            args = (branch,)
-            kwargs = {'depth': herd_depth, 'rebase': rebase, 'fork_remote': fork_remote}
-        elif tag:
-            command = 'herd_tag'
-            args = (tag,)
-            kwargs = {'depth': herd_depth, 'rebase': rebase}
-        else:
-            command = 'herd'
-            args = ()
-            kwargs = {'depth': herd_depth, 'rebase': rebase}
 
         if self.fork is None:
-            getattr(repo, command)(self._url(), *args, **kwargs)
+            if branch:
+                repo.herd_branch(self._url(), branch, depth=herd_depth, rebase=rebase)
+            elif tag:
+                repo.herd_tag(self._url(), tag, depth=herd_depth, rebase=rebase)
+            else:
+                repo.herd(self._url(), depth=herd_depth, rebase=rebase)
             return
 
         self._print(self.fork.status())
         repo.configure_remotes(self.remote, self._url(), self.fork.remote, self.fork.url())
 
         self._print(fmt.fork_string(self.output_name()))
-        kwargs['depth'] = 0  # TODO: Can this be removed?
-        getattr(repo, command)(self._url(), *args, **kwargs)
+        # Modify repo to prefer fork
+        repo.default_ref = self.fork.ref
+        repo.remote = self.fork.remote
+        if branch:
+            repo.herd_branch(self.fork.url(), branch, depth=herd_depth, rebase=rebase)
+        elif tag:
+            repo.herd_tag(self.fork.url(), tag, depth=herd_depth, rebase=rebase)
+        else:
+            repo.herd(self.fork.url(), depth=herd_depth, rebase=rebase)
 
-        self._print(fmt.fork_string(self.fork.name))
-        repo.herd_remote(self.fork.url(), self.fork.remote, branch=branch)
+        self._print(fmt.fork_string(self.name))
+        # Restore repo configuration
+        repo.default_ref = self.ref
+        repo.remote = self.remote
+        repo.herd_remote(self._url(), self.remote, branch=branch)
 
     def is_dirty(self) -> bool:
         """Check if project is dirty
@@ -366,6 +370,10 @@ def prune(self, branch: str, force: bool = False,
         repo = ProjectRepo(self.full_path(), self.remote, self.ref)
 
         if local and repo.existing_local_branch(branch):
+            if self.fork:
+                # Modify repo to prefer fork
+                repo.default_ref = self.fork.ref
+                repo.remote = self.fork.remote
             repo.prune_branch_local(branch, force)
 
         if remote:
@@ -427,6 +435,8 @@ def run(self, commands: List[str], ignore_errors: bool, parallel: bool = False)
 
         if self.fork:
             forall_env['FORK_REMOTE'] = self.fork.remote
+            forall_env['FORK_NAME'] = self.fork.name
+            forall_env['FORK_REF'] = self.fork.ref
 
         for cmd in commands:
             self._run_forall_command(cmd, forall_env, ignore_errors, parallel)

test/scripts/llvm/forks.sh

@@ -55,12 +55,3 @@ cd "$LLVM_EXAMPLE_DIR" || exit 1
 ./init.sh || exit 1
 
 "$TEST_SCRIPT_DIR/llvm/write_forks.sh" $1 || exit 1
-
-test_forks_env() {
-    echo "TEST: Fork remote environment variable in script"
-    $COMMAND forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script_env_fork.sh" -p "llvm-mirror/clang" || exit 1
-    $COMMAND forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script_env_fork.sh" -p "llvm-mirror/llvm" && exit 1
-    echo "TEST: Fork remote environment variable in command"
-    $COMMAND forall $PARALLEL -c 'if [ $PROJECT_REMOTE != upstream ]; then exit 1; fi' -p 'llvm-mirror/clang' || exit 1
-    $COMMAND forall $PARALLEL -c 'if [ $FORK_REMOTE != origin ]; then exit 1; fi' -p 'llvm-mirror/clang' || exit 1
-}

test/scripts/misc/forks.sh

@@ -11,29 +11,40 @@ export project_paths=( 'djinni' \
                        'sox' )
 
 export projects=( 'dropbox/djinni' \
-                  'external/gyp' \
+                  'gyp' \
                   'p/sox/code' )
 
-export fork_paths=( 'djinni' \
-                    'gyp' \
-                    'sox' )
-
-export fork_projects=( 'dropbox/djinni' \
-                       'external/gyp' \
-                       'p/sox/code' )
-
 print_double_separator
 echo "TEST: Test clowder sources"
 cd "$MISC_EXAMPLE_DIR" || exit 1
 ./clean.sh
 ./copy-cache.sh
 
 test_forks_env() {
-    echo 'TODO'
     # echo "TEST: Fork remote environment variable in script"
-    # $COMMAND forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script_env_fork.sh" -p "llvm-mirror/clang" || exit 1
-    # $COMMAND forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script_env_fork.sh" -p "llvm-mirror/llvm" && exit 1
+    $COMMAND forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script_env_fork.sh" -p "gyp" || exit 1
+    $COMMAND forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script_env_fork.sh" -p "dropbox/djinni" && exit 1
     # echo "TEST: Fork remote environment variable in command"
-    # $COMMAND forall $PARALLEL -c 'if [ $PROJECT_REMOTE != upstream ]; then exit 1; fi' -p 'llvm-mirror/clang' || exit 1
-    # $COMMAND forall $PARALLEL -c 'if [ $FORK_REMOTE != origin ]; then exit 1; fi' -p 'llvm-mirror/clang' || exit 1
+    $COMMAND forall $PARALLEL -c 'if [ $PROJECT_REMOTE != upstream ]; then exit 1; fi' -p 'gyp' || exit 1
+    $COMMAND forall $PARALLEL -c 'if [ $FORK_REMOTE != origin ]; then exit 1; fi' -p 'gyp' || exit 1
+}
+test_forks_env
+
+./clean.sh
+./copy-cache.sh
+
+test_fork_herd() {
+    $COMMAND herd $PARALLEL || exit 1
+    pushd 'gyp' || exit 1
+    test_tracking_branch_exists 'fork-branch'
+    fork_branch_commit=''bd11dd1c51ef17592384df927c47023071639f96''
+    test_commit $fork_branch_commit
+    git pull upstream master
+    test_not_commit $fork_branch_commit
+    popd || exit 1
 }
+test_fork_gyp
+
+if [ "$ACCESS_LEVEL" == "write" ]; then
+    "$TEST_SCRIPT_DIR/misc/write_forks.sh" $1
+fi