Merge pull request #510 from JrGoodle/lfs

Add git lfs support

.github/workflows/cancel.yml

@@ -0,0 +1,14 @@
+name: Cancel
+
+on: [push]
+
+jobs:
+  cancel:
+    name: 'Cancel Previous Runs'
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    steps:
+      - uses: styfle/cancel-workflow-action@0.3.1
+        with:
+          workflow_id: 1302111, 1302112, 1302113, 1302114, 1302115, 1302116, 1302117, 1302118, 1302119, 1302120, 1302121, 1302122, 1302123, 1302124, 1302125, 1302126, 1302127, 1302128, 1302129, 1302130, 1302131, 1302132, 1302133, 1302134, 1302135, 1302136, 1302137, 1304392
+          access_token: ${{ github.token }}

.github/workflows/ci-cats-lfs.yml

@@ -0,0 +1,55 @@
+name: 'CI - cats lfs'
+
+on:
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - 'docs/**'
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  cats_lfs:
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
+      OS_NAME: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest] # TODO: windows-latest
+        python-version: ['3.6', '3.7', '3.8']
+        exclude:
+          - os: macos-latest
+            python-version: '3.6'
+          - os: macos-latest
+            python-version: '3.8'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Setup git config
+      run:
+        git config --global user.email "joe@polka.cat";
+        git config --global user.name "GitHub Actions Workflow"
+    - name: Install clowder requirements
+      run: pip3 install -r src/requirements.txt
+    - name: CI before script
+      run: script/ci_before
+    - name: Install clowder
+      run: script/update
+    - name: Install clowder test
+      run: script/test
+    - name: Test lfs commands
+      run: clowder-test -c cats lfs
+    - name: CI after script
+      run: script/ci_after

clowder_test/clowder_test/cli/cats_controller.py

@@ -138,6 +138,14 @@ def link(self) -> None:
 
         self._execute_command('./link.sh', self.path)
 
+    @expose(
+        help='Run cats lfs tests'
+    )
+    def lfs(self) -> None:
+        """clowder cats lfs tests"""
+
+        self._execute_command('./lfs.sh', self.path)
+
     @expose(
         help='Run cats prune tests'
     )

src/clowder/clowder.schema.json

@@ -53,7 +53,13 @@
     },
     "recursive": {
       "description": "Whether to clone project submodules recursively",
-      "type": "boolean"
+      "type": "boolean",
+      "default": false
+    },
+    "lfs": {
+      "description": "Whether repository uses git lfs",
+      "type": "boolean",
+      "default": false
     },
     "branch": {
       "description": "Name of the Git branch to track for this project. If not supplied the ref specified in the defaults is used if applicable, else the default branch 'master' is used.",
@@ -226,6 +232,9 @@
             },
             "recursive": {
               "$ref": "#/definitions/recursive"
+            },
+            "lfs": {
+              "$ref": "#/definitions/lfs"
             }
           }
         },
@@ -271,6 +280,9 @@
             },
             "recursive": {
               "$ref": "#/definitions/recursive"
+            },
+            "lfs": {
+              "$ref": "#/definitions/lfs"
             }
           }
         },
@@ -316,6 +328,9 @@
             },
             "recursive": {
               "$ref": "#/definitions/recursive"
+            },
+            "lfs": {
+              "$ref": "#/definitions/lfs"
             }
           }
         }
@@ -358,6 +373,9 @@
             },
             "depth": {
               "$ref": "#/definitions/depth"
+            },
+            "lfs": {
+              "$ref": "#/definitions/lfs"
             }
           }
         },
@@ -385,6 +403,9 @@
             },
             "depth": {
               "$ref": "#/definitions/depth"
+            },
+            "lfs": {
+              "$ref": "#/definitions/lfs"
             }
           }
         },

src/clowder/git/repo.py

@@ -269,6 +269,20 @@ def get_current_timestamp(self) -> str:
         except (KeyboardInterrupt, SystemExit):
             self._exit()
 
+    def install_lfs_hooks(self):
+        """Install git lfs hooks"""
+
+        self._print(" - Update git lfs hooks")
+        try:
+            self.repo.git.lfs('install')
+        except GitError as err:
+            message = colored(' - Failed to update git lfs hooks', 'red')
+            self._print(message)
+            self._print(fmt.error(err))
+            self._exit(fmt.error(err))
+        except (KeyboardInterrupt, SystemExit):
+            self._exit()
+
     def is_detached(self, print_output: bool = False) -> bool:
         """Check if HEAD is detached
 
@@ -297,6 +311,23 @@ def is_dirty(self) -> bool:
 
         return self.repo.is_dirty() or self._is_rebase_in_progress() or self._has_untracked_files()
 
+    def is_lfs_installed(self) -> bool:
+        """Check whether git lfs hooks are installed
+
+        :return: True, if lfs hooks are installed
+        :rtype: bool
+        """
+
+        try:
+            # FIXME: Probably need to inspect .git hooks
+            self.repo.git.config('--get', 'filter.lfs.smudge')
+            self.repo.git.config('--get', 'filter.lfs.clean')
+            self.repo.git.config('--get', 'filter.lfs.process')
+        except GitError:
+            return False
+        else:
+            return True
+
     def new_commits(self, upstream: bool = False) -> int:
         """Returns the number of new commits
 
@@ -367,6 +398,20 @@ def pull(self) -> None:
         except (KeyboardInterrupt, SystemExit):
             self._exit()
 
+    def pull_lfs(self) -> None:
+        """Pull lfs files"""
+
+        try:
+            self._print(' - Pull git lfs files')
+            self.repo.git.lfs('pull')
+        except GitError as err:
+            message = colored(' - Failed to pull git lfs files', 'red')
+            self._print(message)
+            self._print(fmt.error(err))
+            self._exit(message)
+        except (KeyboardInterrupt, SystemExit):
+            self._exit()
+
     @not_detached
     def push(self) -> None:
         """Push changes"""

src/clowder/model/defaults.py

@@ -24,6 +24,7 @@ class Defaults(object):
     :ivar int depth: Default depth
     :ivar bool recursive: Default recursive value
     :ivar str timestamp_author: Default timestamp author
+    :ivar bool lfs: Default git lfs value
     """
 
     def __init__(self, defaults: dict):
@@ -38,6 +39,7 @@ def __init__(self, defaults: dict):
         self.depth = defaults.get("depth", 0)
         self.recursive = defaults.get("recursive", False)
         self.timestamp_author = defaults.get("timestamp_author", None)
+        self.lfs = defaults.get("lfs", False)
 
         self.branch = defaults.get("branch", None)
         self.tag = defaults.get("tag", None)

src/clowder/model/project.py

@@ -74,6 +74,7 @@ def __init__(self, project: dict, defaults: Defaults, sources: Tuple[Source, ...
         self.depth = project.get('depth', defaults.depth)
         self.recursive = project.get('recursive', defaults.recursive)
         self._timestamp_author = project.get('timestamp_author', defaults.timestamp_author)
+        self._lfs = project.get('lfs', defaults.lfs)
         self._print_output = True
 
         self._branch = project.get("branch", None)
@@ -136,14 +137,12 @@ def branch(self, local: bool = False, remote: bool = False) -> None:
         if self.fork is None:
             if local:
                 repo.print_local_branches()
-
             if remote:
                 repo.print_remote_branches()
             return
 
         if local:
             repo.print_local_branches()
-
         if remote:
             self._print(fmt.fork_string(self.fork.name))
             # Modify repo to prefer fork
@@ -157,15 +156,16 @@ def branch(self, local: bool = False, remote: bool = False) -> None:
             repo.remote = self.remote
             repo.print_remote_branches()
 
-
     @project_repo_exists
     def checkout(self, branch: str) -> None:
         """Checkout branch
 
         :param str branch: Branch to check out
         """
 
-        self._repo(self.recursive).checkout(branch, allow_failure=True)
+        repo = self._repo(self.recursive)
+        repo.checkout(branch, allow_failure=True)
+        self._pull_lfs(repo)
 
     @project_repo_exists
     def clean(self, args: str = '', recursive: bool = False) -> None:
@@ -330,12 +330,15 @@ def herd(self, branch: Optional[str] = None, tag: Optional[str] = None, depth: O
 
         if self.fork is None:
             self._print(self.status())
+
             if branch:
                 repo.herd_branch(self._url(), branch, depth=herd_depth, rebase=rebase)
             elif tag:
                 repo.herd_tag(self._url(), tag, depth=herd_depth, rebase=rebase)
             else:
                 repo.herd(self._url(), depth=herd_depth, rebase=rebase)
+            self._pull_lfs(repo)
+
             return
 
         self._print(self.fork.status())
@@ -351,6 +354,7 @@ def herd(self, branch: Optional[str] = None, tag: Optional[str] = None, depth: O
             repo.herd_tag(self.fork.url(), tag, depth=herd_depth, rebase=rebase)
         else:
             repo.herd(self.fork.url(), depth=herd_depth, rebase=rebase)
+        self._pull_lfs(repo)
 
         self._print(fmt.fork_string(self.name))
         # Restore repo configuration
@@ -429,9 +433,11 @@ def reset(self, timestamp: Optional[str] = None, parallel: bool = False) -> None
         if self.fork is None:
             if timestamp:
                 repo.reset_timestamp(timestamp, self._timestamp_author, self.ref)
+                self._pull_lfs(repo)
                 return
 
             repo.reset(depth=self.depth)
+            self._pull_lfs(repo)
             return
 
         self._print(self.fork.status())
@@ -440,9 +446,11 @@ def reset(self, timestamp: Optional[str] = None, parallel: bool = False) -> None
         self._print(fmt.fork_string(self.fork.name))
         if timestamp:
             repo.reset_timestamp(timestamp, self._timestamp_author, self.ref)
+            self._pull_lfs(repo)
             return
 
         repo.reset()
+        self._pull_lfs(repo)
 
     def run(self, commands: List[str], ignore_errors: bool, parallel: bool = False) -> None:
         """Run commands or script in project directory
@@ -513,6 +521,17 @@ def stash(self) -> None:
             repo = ProjectRepo(self.full_path(), self.remote, self.ref)
             repo.stash()
 
+    def _pull_lfs(self, repo: ProjectRepo) -> None:
+        """Check if git lfs is installed and if not install them
+
+        :param ProjectRepo repo: Repo object
+        """
+        if not self._lfs:
+            return
+
+        repo.install_lfs_hooks()
+        repo.pull_lfs()
+
     def _print(self, val: str) -> None:
         """Print output if self._print_output is True