Addition of wgit add and wgit commit functionalities. Includes refact…

…ors and new classes. (#1002)

* [feat] Adds the implementaion for the wgit add functionality, with sha1 hash creation, reference tracking, dependency graph creation and all related functionalities for the wgit add method.

* [feat] Adds the wgit add and wgit commit functionalities and major refactors.

1. Adds the wgit add and wgit commit functionalities to the api.
2. Introduces a new PyGit class that wraps the internal .wgit/.git repo.
3. Refactors the Repo class in the api, and introduces some methods.
4. .Refactors all the classes which no longer uses @staticmethods and now uses object istances instead.
5.  Moved many of the directory path handling code from os.path to pathlib library.

* [Feat] Combines the Repo and Weigit classes. Separate claases into separate modules.

1. Combines the functionalities of the WeiGit and Repo class into a single WeiGitRepo class.
2. Classes are now separated into their own modules.
3. Moved some functions and staticmethod to utils.
4. Adds a range of tests for add and commit functionalities of weigit.

* [fix] adds a new test to the ci_test_list_3

* [fix] test fix

* [fix] test fix

* [Feat] Directory restructuring, type checking and some standardization
1. Restructured the directory and moved wgit to fairscale/experimental/wgit so that it can be found as a package when pip installed.
2. Added a range of type checking
3. Some refactors

* [Feat][Refactor] Directory restructuring, test addition and type checking
1. Restructed the test directory
2. Added and modified a few wgit tests.
3. Added some type checking to the code

* test fix

* "setup fix and repo checking added in cli"

* [Feat] Better initialization and error handling for init and wgit subcommands. Test reorg.

* [refactor] Changes in classes, encapsulation and addition of PyGit test.

* [Feat][Refactor]
1. Changed some class method arguments for better encapsulation for Sha1_store.
2. Moved sha1 hash calculation within sha1_store.
3. Some standardization and code clean up of unnecessary snippets.
4. Added new tests for the PyGit and Sha1_Store class.

.gitignore

@@ -13,7 +13,7 @@
 .testmondata
 
 # experimental weigit
-experimental/.gitignore
+fairscale/experimental/wgit/dev
 .wgit
 
 # Build and release

experimental/wgit/__init__.py → fairscale/experimental/wgit/__init__.py

@@ -3,7 +3,7 @@
 # This source code is licensed under the BSD license found in the
 # LICENSE file in the root directory of this source tree.
 
-from .cli import main
-from .weigit_api import WeiGit
+from .repo import Repo
+from .version import __version_tuple__
 
-__version__ = "0.0.1"
+__version__ = ".".join([str(x) for x in __version_tuple__])

experimental/wgit/cli.py → fairscale/experimental/wgit/cli.py

@@ -4,12 +4,13 @@
 # LICENSE file in the root directory of this source tree.
 
 import argparse
+from pathlib import Path
+from typing import List
 
-import experimental.wgit as wgit
-import experimental.wgit.weigit_api as weigit_api
+from . import Repo, version
 
 
-def main(argv=None):
+def main(argv: List[str] = None) -> None:
     desc = "WeiGit: A git-like tool for model weight tracking"
 
     # top level parser and corresponding subparser
@@ -71,25 +72,30 @@ def main(argv=None):
     args = parser.parse_args(argv)
 
     if args.command == "init":
-        weigit = weigit_api.WeiGit()
+        repo = Repo(Path.cwd(), init=True)
 
     if args.command == "add":
-        weigit_api.WeiGit.add(args.add)
+        repo = Repo(Path.cwd())
+        repo.add(args.add)
 
     if args.command == "status":
-        weigit_api.WeiGit.status()
+        repo = Repo(Path.cwd())
+        repo.status()
 
     if args.command == "log":
-        weigit_api.WeiGit.log(args.file)
+        repo = Repo(Path.cwd())
+        repo.log(args.file)
 
     if args.command == "commit":
-        weigit_api.WeiGit.commit(args.message)
+        repo = Repo(Path.cwd())
+        repo.commit(args.message)
 
     if args.command == "checkout":
-        weigit_api.WeiGit.checkout()
+        repo = Repo(Path.cwd())
+        repo.checkout(args.checkout)
 
     if args.command == "version":
-        print(wgit.__version__)
+        print(".".join([str(x) for x in version.__version_tuple__]))
 
 
 if __name__ == "__main__":

fairscale/experimental/wgit/pygit.py

@@ -0,0 +1,100 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the BSD license found in the
+# LICENSE file in the root directory of this source tree.
+
+from pathlib import Path
+from typing import List
+
+import pygit2
+
+
+class PyGit:
+    def __init__(
+        self,
+        parent_path: Path,
+        gitignore: List = list(),
+        name: str = "user",
+        email: str = "user@email.com",
+    ) -> None:
+        """
+        PyGit class to wrap the wgit/.git repo and interact with the git repo.
+
+        Args:
+        parent_path: Has to be the full path of the parent!
+        """
+        # Find if a git repo exists within .wgit repo:
+        # If exists: then discover it and set the self.gitrepo path to its path
+        self._parent_path = parent_path
+        self.name = name
+        self.email = email
+
+        git_repo_found = pygit2.discover_repository(self._parent_path)
+
+        if git_repo_found:
+            # grab the parent dir of this git repo
+            git_repo = Path(pygit2.discover_repository(self._parent_path))
+            pygit_parent_p = git_repo.parent.absolute()
+
+            # Check If the parent dir is a .wgit dir. If the .wgit is a git repo
+            # just wrap the existing git repo with pygit2.Repository class
+            if pygit_parent_p == self._parent_path:
+                self.repo = pygit2.Repository(str(self._parent_path))
+                self.path = self._parent_path.joinpath(".git")
+            else:
+                # if the parent is not a .wgit repo,
+                # then the found-repo is a different git repo. Init new .wgit/.git
+                self._init_wgit_git(gitignore)
+        else:
+            # no git repo found, make .wgit a git repo
+            self._init_wgit_git(gitignore)
+
+    def _init_wgit_git(self, gitignore: List) -> None:
+        """Initializes a .git within .wgit directory, making it a git repo."""
+        self.repo = pygit2.init_repository(str(self._parent_path), False)
+        self.path = self._parent_path.joinpath(".git")
+
+        # create and populate a .gitignore
+        self._parent_path.joinpath(".gitignore").touch(exist_ok=False)
+
+        with open(self._parent_path.joinpath(".gitignore"), "a") as file:
+            for item in gitignore:
+                file.write(f"{item}\n")
+
+    def add(self) -> None:
+        """git add all the untracked files not in gitignore, to the .wgit/.git repo."""
+        # If .wgit is git repo, add all the files in .wgit not being ignored to git
+        if self._exists:
+            self.repo.index.add_all()
+            self.repo.index.write()
+
+    def commit(self, message: str) -> None:
+        """git commit the staged changes to the .wgit/.git repo."""
+        # If .wgit is git repo, commit the staged files to git
+        if self._exists:
+            # if no commit exists, set ref to HEAD and parents to empty
+            try:
+                ref = self.repo.head.name
+                parents = [self.repo.head.target]
+            except pygit2.GitError:
+                ref = "HEAD"
+                parents = []
+
+            author = pygit2.Signature(self.name, self.email)
+            committer = pygit2.Signature(self.name, self.email)
+            tree = self.repo.index.write_tree()
+            self.repo.create_commit(ref, author, committer, message, tree, parents)
+
+    @property
+    def _exists(self) -> bool:
+        """returns True if wgit is a git repository"""
+        return self._parent_path == Path(self.repo.path).parent
+
+    @property
+    def _path(self) -> str:
+        """returns the path of the git repository PyGit is wrapped around"""
+        return self.repo.path
+
+    def status(self) -> None:
+        """Print the status of the git repo"""
+        print(self.repo.status())