Merge pull request #121 from datmo/sharing

Enable sharing between locations

README.md

@@ -160,22 +160,38 @@ In order to run the above code you can do the following.
 
         $ datmo snapshot ls
 
-## Sharing (Beta)
-Although datmo is made to track your changes locally, you can share a project with your
-friends by doing the following (this is shown only for git, if you are using another git 
-tracking tool, you can likely do something similar). NOTE: If your files are too big or 
+## Sharing (Workaround)
+**DISCLAIMER:** This is not currently an officially supported option and only works for 
+file-based storage layers (as set in the configuration) as a workaround to share datmo projects. 
+
+Although datmo is made to track changes locally, you can share a project by pushing to a remote 
+server by doing the following (this is shown only for git, if you are using another SCM 
+tracking tool, you can likely do something similar). If your files are too big or 
 cannot be added to SCM then this may not work for you. 
+
+The below has been tested on BASH terminals only. If you are using another terminal, you 
+may run into some errors. 
+
+### Push to remote
 ```
-$ git add -f .datmo/*
-$ git commit -m "my_message"
-$ git push 
-$ git push origin +refs/datmo/*:refs/datmo/*
+$ git add -f .datmo/*  # add in .datmo to your scm
+$ git commit -m "adding .datmo to tracking"  # commit it to your scm
+$ git push  # push to remote
+$ git push origin +refs/datmo/*:refs/datmo/*  # push datmo refs to remote
 ```
 The above will allow you to share datmo results and entities with yourself or others on 
 other machines. NOTE: you will have to remove .datmo/ from tracking to start using datmo
-on the other machine. To do that you can use the commands below
+on the other machine or another location. See the instructions below to see how to replicate
+it at another location
+
+### Pull from remote
+```
+$ git clone YOUR_REMOTE_URL
+$ cd YOUR_REPO 
+$ echo '.datmo/*' > .git/info/exclude  # include .datmo into your .git exclude
+$ git rm -r --cached .datmo  # remove cached versions of .datmo from scm
+$ git commit -m "removed .datmo from tracking"  # clean up your scm so datmo can work 
+$ git pull origin +refs/datmo/*:refs/datmo/*  # pull datmo refs from remote
+$ datmo init  # This enables datmo in the new location. If you enter blanks, no project information will be updated
 ```
-$ git rm -r --cached
-$ git add .
-$ git commit -m "removed .datmo from tracking"
-```
+If you are interested in sharing using the datmo protocol, you can visit [Datmo's website](https://datmo.com/product)

datmo/core/controller/code/driver/git.py

@@ -413,8 +413,7 @@ def exists_commit(self, commit_id):
                 stderr=subprocess.PIPE,
                 cwd=self.filepath)
             stdout, stderr = process.communicate()
-            stdout, stderr = stdout.decode('utf-8'), stderr.decode('utf-8')
-            if process.returncode > 0 or "fatal" in stdout:
+            if process.returncode > 0 or "fatal" in str(stdout):
                 return False
         except subprocess.CalledProcessError:
             return False

datmo/core/controller/code/driver/tests/test_git.py

@@ -136,24 +136,24 @@ def test_exists_datmo_files_in_worktree(self):
         result = self.git_code_manager.exists_datmo_files_in_worktree()
         assert result == True
 
-    def test_clone(self):
-        result = self.git_code_manager.clone(
-            "https://github.com/datmo/hello-world.git", mode="https")
-        assert result and os.path.exists(
-            os.path.join(self.temp_dir, "hello-world", ".git"))
-        result = self.git_code_manager.clone(
-            "https://github.com/datmo/hello-world.git",
-            repo_name="hello-world-2",
-            mode="http")
-        assert result and os.path.exists(
-            os.path.join(self.temp_dir, "hello-world-2", ".git"))
-        if self.git_code_manager.git_host_manager.ssh_enabled:
-            result = self.git_code_manager.clone(
-                "https://github.com/datmo/hello-world.git",
-                repo_name="hello-world-3",
-                mode="ssh")
-            assert result and os.path.exists(
-                os.path.join(self.temp_dir, "hello-world-3", ".git"))
+    # def test_clone(self):
+    #     result = self.git_code_manager.clone(
+    #         "https://github.com/datmo/hello-world.git", mode="https")
+    #     assert result and os.path.exists(
+    #         os.path.join(self.temp_dir, "hello-world", ".git"))
+    #     result = self.git_code_manager.clone(
+    #         "https://github.com/datmo/hello-world.git",
+    #         repo_name="hello-world-2",
+    #         mode="http")
+    #     assert result and os.path.exists(
+    #         os.path.join(self.temp_dir, "hello-world-2", ".git"))
+    #     if self.git_code_manager.git_host_manager.ssh_enabled:
+    #         result = self.git_code_manager.clone(
+    #             "https://github.com/datmo/hello-world.git",
+    #             repo_name="hello-world-3",
+    #             mode="ssh")
+    #         assert result and os.path.exists(
+    #             os.path.join(self.temp_dir, "hello-world-3", ".git"))
 
     def test_parse_git_url(self):
         parsed = self.git_code_manager._parse_git_url(

datmo/core/controller/environment/driver/dockerenv.py

@@ -194,9 +194,23 @@ def get_tags_for_docker_repository(self, repo_name):
             List of tags available for that docker repo
         """
         docker_repository_tag_cmd = "wget -q https://registry.hub.docker.com/v1/repositories/" + repo_name + "/tags -O -"
-        string_repository_tags = subprocess.check_output(
-            docker_repository_tag_cmd, shell=True)
-        string_repository_tags = string_repository_tags.decode().strip()
+        try:
+            process = subprocess.Popen(
+                docker_repository_tag_cmd,
+                shell=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            stdout, stderr = process.communicate()
+            if process.returncode > 0:
+                raise EnvironmentExecutionException(
+                    __("error",
+                       "controller.environment.driver.docker.get_tags",
+                       str(stderr)))
+            string_repository_tags = stdout.decode().strip()
+        except subprocess.CalledProcessError as e:
+            raise EnvironmentExecutionException(
+                __("error", "controller.environment.driver.docker.get_tags",
+                   str(e)))
         repository_tags = ast.literal_eval(string_repository_tags)
         list_tag_names = []
         for repository_tag in repository_tags:
@@ -273,8 +287,17 @@ def remove_image(self, image_id_or_name, force=False):
             else:
                 docker_image_remove_cmd = list(self.prefix)
                 docker_image_remove_cmd.extend(["rmi", image_id_or_name])
-            subprocess.check_output(docker_image_remove_cmd).decode().strip()
-        except Exception as e:
+            process = subprocess.Popen(
+                docker_image_remove_cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            stdout, stderr = process.communicate()
+            if process.returncode > 0:
+                raise EnvironmentExecutionException(
+                    __("error",
+                       "controller.environment.driver.docker.remove_image",
+                       str(stderr)))
+        except subprocess.CalledProcessError as e:
             raise EnvironmentExecutionException(
                 __("error",
                    "controller.environment.driver.docker.remove_image",
@@ -425,10 +448,18 @@ def run_container(self,
                            str(docker_shell_cmd_list)))
                 list_process_cmd = list(self.prefix)
                 list_process_cmd.extend(["ps", "-q", "-l"])
-                container_id = subprocess.check_output(list_process_cmd)
-                container_id = container_id.decode().strip()
-
-        except Exception as e:
+                process = subprocess.Popen(
+                    list_process_cmd,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE)
+                stdout, stderr = process.communicate()
+                if process.returncode > 0:
+                    raise EnvironmentExecutionException(
+                        __("error",
+                           "controller.environment.driver.docker.run_container",
+                           str(stderr)))
+                container_id = stdout.decode().strip()
+        except subprocess.CalledProcessError as e:
             raise EnvironmentExecutionException(
                 __("error",
                    "controller.environment.driver.docker.run_container",
@@ -454,8 +485,17 @@ def stop_container(self, container_id):
         try:
             docker_container_stop_cmd = list(self.prefix)
             docker_container_stop_cmd.extend(["stop", container_id])
-            subprocess.check_output(docker_container_stop_cmd).decode().strip()
-        except Exception as e:
+            process = subprocess.Popen(
+                docker_container_stop_cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            stdout, stderr = process.communicate()
+            if process.returncode > 0:
+                raise EnvironmentExecutionException(
+                    __("error",
+                       "controller.environment.driver.docker.stop_container",
+                       str(stderr)))
+        except subprocess.CalledProcessError as e:
             raise EnvironmentExecutionException(
                 __("error",
                    "controller.environment.driver.docker.stop_container",
@@ -470,9 +510,17 @@ def remove_container(self, container_id, force=False):
                     ["rm", "-f", container_id])
             else:
                 docker_container_remove_cmd_list.extend(["rm", container_id])
-            subprocess.check_output(
-                docker_container_remove_cmd_list).decode().strip()
-        except Exception as e:
+            process = subprocess.Popen(
+                docker_container_remove_cmd_list,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            stdout, stderr = process.communicate()
+            if process.returncode > 0:
+                raise EnvironmentExecutionException(
+                    __("error",
+                       "controller.environment.driver.docker.remove_container",
+                       str(stderr)))
+        except subprocess.CalledProcessError as e:
             raise EnvironmentExecutionException(
                 __("error",
                    "controller.environment.driver.docker.remove_container",
@@ -540,12 +588,17 @@ def stop_remove_containers_by_term(self, term, force=False):
 
             running_docker_container_cmd_str = str(
                 " ".join(running_docker_container_cmd_list))
-            output = subprocess.Popen(
+            process = subprocess.Popen(
                 running_docker_container_cmd_str,
                 shell=True,
-                stdout=subprocess.PIPE)
-            out_list_cmd, err_list_cmd = output.communicate()
-
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            out_list_cmd, err_list_cmd = process.communicate()
+            if process.returncode > 0:
+                raise EnvironmentExecutionException(
+                    __("error",
+                       "controller.environment.driver.docker.stop_remove_containers_by_term",
+                       str(err_list_cmd)))
             # checking for running container id before stopping any
             if out_list_cmd:
                 docker_container_stop_cmd_list = list(self.prefix)
@@ -554,17 +607,24 @@ def stop_remove_containers_by_term(self, term, force=False):
                                                  [")"]
                 docker_container_stop_cmd_str = str(
                     " ".join(docker_container_stop_cmd_list))
-                output = subprocess.Popen(
+                process = subprocess.Popen(
                     docker_container_stop_cmd_str,
                     shell=True,
-                    stdout=subprocess.PIPE)
-                _, _ = output.communicate()
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE)
+                _, _ = process.communicate()
                 # rechecking for container id after stopping them to ensure no errors
-                output = subprocess.Popen(
+                process = subprocess.Popen(
                     running_docker_container_cmd_str,
                     shell=True,
-                    stdout=subprocess.PIPE)
-                out_list_cmd, _ = output.communicate()
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE)
+                out_list_cmd, err_list_cmd = process.communicate()
+                if process.returncode > 0:
+                    raise EnvironmentExecutionException(
+                        __("error",
+                           "controller.environment.driver.docker.stop_remove_containers_by_term",
+                           str(err_list_cmd)))
                 if out_list_cmd:
                     docker_container_remove_cmd_list = list(self.prefix)
                     if force:
@@ -577,12 +637,18 @@ def stop_remove_containers_by_term(self, term, force=False):
                                                            [")"]
                     docker_container_remove_cmd_str = str(
                         " ".join(docker_container_remove_cmd_list))
-                    output = subprocess.Popen(
+                    process = subprocess.Popen(
                         docker_container_remove_cmd_str,
                         shell=True,
-                        stdout=subprocess.PIPE)
-                    _, _ = output.communicate()
-        except Exception as e:
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE)
+                    _, err_list_cmd = process.communicate()
+                    if process.returncode > 0:
+                        raise EnvironmentExecutionException(
+                            __("error",
+                               "controller.environment.driver.docker.stop_remove_containers_by_term",
+                               str(err_list_cmd)))
+        except subprocess.CalledProcessError as e:
             raise EnvironmentExecutionException(
                 __("error",
                    "controller.environment.driver.docker.stop_remove_containers_by_term",
@@ -605,17 +671,25 @@ def create_requirements_file(self, execpath="pipreqs"):
 
         Raises
         ------
-        EnvironmentDoesNotExist
-            no python requirements found for environment
         EnvironmentRequirementsCreateException
             error in running pipreqs command to extract python requirements
         """
         try:
-            subprocess.check_output(
-                [execpath, self.filepath, "--force"],
-                cwd=self.filepath).strip()
             requirements_filepath = os.path.join(self.filepath,
-                                                 "requirements.txt")
+                                                 "datmorequirements.txt")
+            process = subprocess.Popen(
+                [
+                    execpath, self.filepath, "--force", "--savepath",
+                    requirements_filepath
+                ],
+                cwd=self.filepath,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            stdout, stderr = process.communicate()
+            if process.returncode > 0:
+                raise EnvironmentRequirementsCreateException(
+                    __("error", "controller.environment.requirements.create",
+                       str(stderr)))
         except Exception as e:
             raise EnvironmentRequirementsCreateException(
                 __("error", "controller.environment.requirements.create",

datmo/core/controller/environment/tests/test_environment.py

@@ -170,7 +170,7 @@ def test_create(self):
         assert environment_obj_4.hardware_info
         assert environment_obj_4.unique_hash == file_collection_obj.filehash
         assert os.path.isfile(
-            os.path.join(file_collection_dir, "requirements.txt"))
+            os.path.join(file_collection_dir, "datmorequirements.txt"))
         assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
         assert os.path.isfile(
             os.path.join(file_collection_dir, "datmoDockerfile"))

datmo/core/controller/file/driver/local.py

@@ -98,8 +98,12 @@ def copyfile(filepath, dst_dirpath):
     @property
     def is_initialized(self):
         if self.exists_datmo_file_structure():
-            self._is_initialized = True
-            return self._is_initialized
+            if self.exists_collections_dir():
+                if os.path.isdir(
+                        os.path.join(self.filepath, ".datmo", "collections",
+                                     "d41d8cd98f00b204e9800998ecf8427e")):
+                    self._is_initialized = True
+                    return self._is_initialized
         self._is_initialized = False
         return self._is_initialized
 
@@ -109,6 +113,16 @@ def init(self):
         try:
             # Ensure the Datmo file structure exists
             self.ensure_datmo_file_structure()
+            # Ensure the collections directory exists
+            self.ensure_collections_dir()
+            # Ensure the empty collection exists
+            if not os.path.isdir(
+                    os.path.join(self.filepath, ".datmo", "collections",
+                                 "d41d8cd98f00b204e9800998ecf8427e")):
+                self.create(
+                    os.path.join(".datmo", "collections",
+                                 "d41d8cd98f00b204e9800998ecf8427e"),
+                    directory=True)
         except Exception as e:
             raise FileIOException(
                 __("error", "controller.file.driver.local.init", str(e)))
@@ -232,6 +246,7 @@ def create_collection(self, filepaths):
         collection_path = os.path.join(self.filepath, ".datmo", "collections",
                                        filehash)
         if os.path.isdir(collection_path):
+            shutil.rmtree(temp_collection_path)
             return filehash
             # raise FileStructureException("exception.file.create_collection", {
             #     "exception": "File collection with id already exists."