From a256a03b16b3f96ceb68f5fc635917b6a5d55309 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Thu, 20 Feb 2020 14:30:23 +0100 Subject: [PATCH 01/10] RF: Rip out GitRepo.repo Defer import of GitPython and instantiation to the few pieces in DataLad's tests that need a GitPython repo instance to work. --- datalad/interface/ls.py | 1 - datalad/support/annexrepo.py | 2 - datalad/support/gitrepo.py | 187 +----------------------- datalad/support/tests/test_annexrepo.py | 11 -- datalad/support/tests/test_gitrepo.py | 64 +------- datalad/support/tests/utils.py | 50 ------- datalad/tests/utils.py | 3 +- 7 files changed, 9 insertions(+), 309 deletions(-) diff --git a/datalad/interface/ls.py b/datalad/interface/ls.py index 842e936dd7..0135d77cb0 100644 --- a/datalad/interface/ls.py +++ b/datalad/interface/ls.py @@ -505,7 +505,6 @@ def _ls_dataset(loc, fast=False, recursive=False, all_=False, long_=False): # workaround for explosion of git cat-file --batch processes # https://github.com/datalad/datalad/issues/1888 if dsm.repo is not None: - dsm.repo.repo.close() del dsm.repo dsm.repo = None diff --git a/datalad/support/annexrepo.py b/datalad/support/annexrepo.py index 0e5aa2547d..c75f48ad86 100644 --- a/datalad/support/annexrepo.py +++ b/datalad/support/annexrepo.py @@ -3039,8 +3039,6 @@ def uuid(self): Returns a the annex UUID, if there is any, or `None` otherwise. """ if not self._uuid: - if not self.repo: - return None self._uuid = self.config.get('annex.uuid', default=None) return self._uuid diff --git a/datalad/support/gitrepo.py b/datalad/support/gitrepo.py index 75ab06d9f6..44520ba837 100644 --- a/datalad/support/gitrepo.py +++ b/datalad/support/gitrepo.py @@ -6,9 +6,7 @@ # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## -"""Interface to Git via GitPython - -For further information on GitPython see http://gitpython.readthedocs.org/ +"""Internal low-level interface to Git repositories """ @@ -20,7 +18,6 @@ from locale import getpreferredencoding -from itertools import chain import logging from collections import ( OrderedDict, @@ -46,8 +43,6 @@ from functools import wraps from weakref import WeakValueDictionary -import git as gitpy -from gitdb.exc import BadName from git.exc import ( NoSuchPathError, InvalidGitRepositoryError @@ -96,7 +91,6 @@ FileNotInRepositoryError, GitIgnoreError, InvalidGitReferenceError, - MissingBranchError, OutdatedExternalDependencyWarning, PathKnownToRepositoryError, ) @@ -111,29 +105,12 @@ RepoInterface ) -#from git.objects.blob import Blob - # shortcuts _curdirsep = curdir + sep _pardirsep = pardir + sep lgr = logging.getLogger('datalad.gitrepo') -_lgr_level = lgr.getEffectiveLevel() -if _lgr_level <= 2: - from ..log import LoggerHelper - # Let's also enable gitpy etc debugging - gitpy_lgr = LoggerHelper(logtarget="git").get_initialized_logger() - gitpy_lgr.setLevel(_lgr_level) - gitpy_lgr.propagate = True - -# Override default GitPython's DB backend to talk directly to git so it doesn't -# interfere with possible operations performed by gc/repack -default_git_odbt = gitpy.GitCmdObjectDB - -# TODO: Figure out how GIT_PYTHON_TRACE ('full') is supposed to be used. -# Didn't work as expected on a first try. Probably there is a neatier way to -# log Exceptions from git commands. def to_options(split_single_char_options=True, **kwargs): @@ -375,27 +352,6 @@ def remap_filenames(out): return newfunc -def _remove_empty_items(list_): - """Remove empty entries from list - - This is needed, since some functions of GitPython may convert - an empty entry to '.', when used with a list of paths. - - Parameter: - ---------- - list_: list of str - - Returns - ------- - list of str - """ - if not isinstance(list_, list): - lgr.warning( - "_remove_empty_items() called with non-list type: %s" % type(list_)) - return list_ - return [file_ for file_ in list_ if file_] - - if "2.24.0" <= external_versions["cmd:git"] < "2.25.0": # An unintentional change in Git 2.24.0 led to `ls-files -o` traversing # into untracked submodules when multiple pathspecs are given, returning @@ -416,41 +372,6 @@ def _prune_deeper_repos(repos): return repos -def Repo(*args, **kwargs): - """Factory method around gitpy.Repo to consistently initiate with different - backend - """ - # TODO: This probably doesn't work as intended (or at least not as - # consistently as intended). gitpy.Repo could be instantiated by - # classmethods Repo.init or Repo.clone_from. In these cases 'odbt' - # would be needed as a parameter to these methods instead of the - # constructor. - if 'odbt' not in kwargs: - kwargs['odbt'] = default_git_odbt - return gitpy.Repo(*args, **kwargs) - - -def guard_BadName(func): - """A helper to guard against BadName exception - - Workaround for - https://github.com/gitpython-developers/GitPython/issues/768 - also see https://github.com/datalad/datalad/issues/2550 - Let's try to precommit (to flush anything flushable) and do - it again - """ - - @wraps(func) - def wrapped(repo, *args, **kwargs): - try: - return func(repo, *args, **kwargs) - except BadName: - repo.precommit() - return func(repo, *args, **kwargs) - - return wrapped - - class GitProgress(WitlessProtocol): """Reduced variant of GitPython's RemoteProgress class @@ -928,7 +849,7 @@ def __init__(self, path, url=None, runner=None, create=True, If set to false, an exception is raised in case `path` doesn't exist or doesn't contain a git repository. repo: git.Repo, optional - GitPython's Repo instance to (re)use if provided + This argument is ignored. create_sanity_checks: bool, optional Whether to perform sanity checks during initialization (when `create=True` and target path is not a valid repo already), such as @@ -1021,9 +942,6 @@ def __init__(self, path, url=None, runner=None, create=True, # Could be used to e.g. disable automatic garbage and autopacking # ['-c', 'receive.autogc=0', '-c', 'gc.auto=0'] self._GIT_COMMON_OPTIONS = [] - # actually no need with default GitPython db backend not in memory - # default_git_odbt but still allows for faster testing etc. - # May be eventually we would make it switchable _GIT_COMMON_OPTIONS = [] if git_opts is None: git_opts = {} @@ -1031,18 +949,10 @@ def __init__(self, path, url=None, runner=None, create=True, git_opts.update(kwargs) self.cmd_call_wrapper = runner or GitRunner(cwd=self.path) - self._repo = repo self._cfg = None - if do_create: # we figured it out ealier - self._repo = self._create_empty_repo(path, - create_sanity_checks, **git_opts) - - # inject git options into GitPython's git call wrapper: - # Note: `None` currently can happen, when Runner's protocol prevents - # calls above from being actually executed (DryRunProtocol) - if self._repo is not None: - self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS + if do_create: # we figured it out earlier + self._create_empty_repo(path, create_sanity_checks, **git_opts) # with DryRunProtocol path might still not exist if exists(self.path): @@ -1106,43 +1016,6 @@ def _create_empty_repo(self, path, sanity_checks=True, **kwargs): except CommandError as exc: lgr.error(exc_str(exc)) raise - # we want to return None and have lazy eval take care of - # the rest - return - - @property - def repo(self): - # with DryRunProtocol path not exist - if exists(self.path): - inode = os.stat(self.path).st_ino - else: - inode = None - if self.inode != inode: - - # TODO: - what if we never had any inode? possible? - # - also: call to self._repo and only afterwards checking whether it's None seems strange - - # reset background processes invoked by GitPython: - # it can be that we never got to instantiate a GitPython repo instance - if self._repo: - self._repo.git.clear_cache() - self.inode = inode - - if self._repo is None: - # Note, that this may raise GitCommandError, NoSuchPathError, - # InvalidGitRepositoryError: - self._repo = self.cmd_call_wrapper( - Repo, - self.path) - lgr.log(8, "Using existing Git repository at %s", self.path) - - # inject git options into GitPython's git call wrapper: - # Note: `None` currently can happen, when Runner's protocol prevents - # call of Repo(path) above from being actually executed (DryRunProtocol) - if self._repo is not None: - self._repo.git._persistent_git_options = self._GIT_COMMON_OPTIONS - - return self._repo @classmethod def clone(cls, url, path, *args, clone_options=None, **kwargs): @@ -1270,22 +1143,6 @@ def __del__(self): # unbind possibly bound ConfigManager, to prevent all kinds of weird # stalls etc self._cfg = None - # Make sure to flush pending changes, especially close batch processes - # (internal `git cat-file --batch` by GitPython) - try: - if getattr(self, '_repo', None) is not None and exists(self.path): - # gc might be late, so the (temporary) - # repo doesn't exist on FS anymore - self._repo.git.clear_cache() - # We used to write out the index to flush GitPython's - # state... but such unconditional write is really a workaround - # and does not play nice with read-only operations - permission - # denied etc. So disabled - #if exists(opj(self.path, '.git')): # don't try to write otherwise - # self.repo.index.write() - except (InvalidGitRepositoryError, AttributeError): - # might have being removed and no longer valid or attributes unbound - pass def __repr__(self): return "" % (self.path, type(self)) @@ -1575,9 +1432,6 @@ def remove(self, files, recursive=False, **kwargs): [str] list of successfully removed files. """ - - files = _remove_empty_items(files) - if recursive: kwargs['r'] = True stdout, stderr = self._git_custom_command( @@ -1586,23 +1440,10 @@ def remove(self, files, recursive=False, **kwargs): # output per removed file is expected to be "rm 'PATH'": return [line.strip()[4:-1] for line in stdout.splitlines()] - #return self.repo.git.rm(files, cached=False, **kwargs) - def precommit(self): """Perform pre-commit maintenance tasks """ - # All GitPython commands should take care about flushing index - # whenever they modify it, so we would not care to do anything - # if self.repo is not None and exists(opj(self.path, '.git')): # don't try to write otherwise: - # # flush possibly cached in GitPython changes to index: - # # if self.repo.git: - # # sys.stderr.write("CLEARING\n") - # # self.repo.git.clear_cache() - # self.repo.index.write() - - # Close batched by GitPython git processes etc - # Ref: https://github.com/gitpython-developers/GitPython/issues/718 - self.repo.__del__() + # we used to clean up GitPython here pass @staticmethod @@ -2119,11 +1960,6 @@ def get_remotes(self, with_urls_only=False): remotes : list of str List of names of the remotes """ - - # Note: read directly from config and spare instantiation of gitpy.Repo - # since we need this in AnnexRepo constructor. Furthermore gitpy does it - # pretty much the same way and the use of a Repo instance seems to have - # no reason other than a nice object oriented look. from datalad.utils import unique self.config.reload() @@ -2170,9 +2006,7 @@ def _git_custom_command(self, files, cmd_str, updates_tree=False): """Allows for calling arbitrary commands. - Helper for developing purposes, i.e. to quickly implement git commands - for proof of concept without the need to figure out, how this is done - via GitPython. + The method should be avoided and the call_git*() should be used instead. Parameters ---------- @@ -2435,7 +2269,6 @@ def fetch(self, remote=None, refspec=None, all_=False, git_options=None, ) ) - @guard_BadName def fetch_(self, remote=None, refspec=None, all_=False, git_options=None): """Like `fetch`, but returns a generator""" yield from self._fetch_push_helper( @@ -2966,14 +2799,6 @@ def add_submodule(self, path, name=None, url=None, branch=None): """ if name is None: name = Path(path).as_posix() - # XXX the following should do it, but GitPython will refuse to add a submodule - # unless you specify a URL that is configured as one of its remotes, or you - # specify no URL, but the repo has at least one remote. - # this is stupid, as for us it is valid to not have any remote, because we can - # still obtain the submodule from a future publication location, based on the - # parent - # gitpy.Submodule.add(self.repo, name, path, url=url, branch=branch) - # going git native instead cmd = ['submodule', 'add', '--name', name] if branch is not None: cmd += ['-b', branch] diff --git a/datalad/support/tests/test_annexrepo.py b/datalad/support/tests/test_annexrepo.py index b95a27ff00..274ac1eeeb 100644 --- a/datalad/support/tests/test_annexrepo.py +++ b/datalad/support/tests/test_annexrepo.py @@ -101,7 +101,6 @@ from datalad.support.exceptions import ( AnnexBatchCommandError, CommandError, - CommandNotAvailableError, FileInGitError, FileNotInAnnexError, FileNotInRepositoryError, @@ -112,7 +111,6 @@ OutOfSpaceError, RemoteNotAvailableError, ) -from datalad.support.external_versions import external_versions from datalad.support.gitrepo import GitRepo @@ -123,8 +121,6 @@ AnnexJsonProtocol, ProcessAnnexProgressIndicators, ) -from .utils import check_repo_deals_with_inode_change - @assert_cwd_unchanged @with_testrepos('.*annex.*') @@ -2026,13 +2022,6 @@ def test_AnnexRepo_is_managed_branch(path): ok_(ar.is_managed_branch()) -@with_tempfile(mkdir=True) -@with_tempfile() -def test_AnnexRepo_flyweight_monitoring_inode(path, store): - # testing for issue #1512 - check_repo_deals_with_inode_change(AnnexRepo, path, store) - - @with_tempfile(mkdir=True) def test_fake_is_not_special(path): ar = AnnexRepo(path, create=True) diff --git a/datalad/support/tests/test_gitrepo.py b/datalad/support/tests/test_gitrepo.py index 2a912d295e..2c761e0844 100644 --- a/datalad/support/tests/test_gitrepo.py +++ b/datalad/support/tests/test_gitrepo.py @@ -28,7 +28,6 @@ getpwd, on_windows, rmtree, - unlink, ) from datalad.tests.utils import ( assert_cwd_unchanged, @@ -37,7 +36,6 @@ assert_in_results, assert_not_in, assert_raises, - assert_re_in, create_tree, eq_, get_most_obscure_supported_name, @@ -57,17 +55,11 @@ with_testrepos, with_tree, ) -from datalad.tests.utils_testrepos import BasicAnnexTestRepo - -from datalad.dochelpers import exc_str - from datalad.support.sshconnector import get_connection_hash from datalad.support.gitrepo import ( _normalize_path, - gitpy, GitRepo, - guard_BadName, InvalidGitRepositoryError, normalize_paths, NoSuchPathError, @@ -81,7 +73,6 @@ ) from datalad.support.external_versions import external_versions from datalad.support.protocol import ExecutionTimeProtocol -from .utils import check_repo_deals_with_inode_change @with_tempfile(mkdir=True) @@ -100,8 +91,6 @@ def test_GitRepo_instance_from_clone(src, dst): gr = GitRepo.clone(src, dst) assert_is_instance(gr, GitRepo, "GitRepo was not created.") - assert_is_instance(gr.repo, gitpy.Repo, - "Failed to instantiate GitPython Repo object.") ok_(op.exists(op.join(dst, '.git'))) # do it again should raise ValueError since git will notice there's @@ -970,7 +959,7 @@ def test_submodule_deinit(path): # using force should work: top_repo.deinit_submodule('subm 1', force=True) - ok_(not top_repo.repo.submodule('subm 1').module_exists()) + ok_(not GitRepo.is_valid_repo(str(top_repo.pathobj / 'subm 1'))) @with_testrepos(".*basic_git.*", flavors=['local']) @@ -1155,30 +1144,6 @@ def _get_inodes(repo): # return the original object in second run -@with_tempfile -@with_tempfile -def test_GitRepo_gitpy_injection(path, path2): - from git import GitCommandError - - gr = GitRepo(path, create=True) - gr._GIT_COMMON_OPTIONS.extend(['test-option']) - - with assert_raises(GitCommandError) as cme: - gr.repo.git.unknown_git_command() - assert_in('test-option', exc_str(cme.exception)) - - # once set, these option should be persistent across git calls: - with assert_raises(GitCommandError) as cme: - gr.repo.git.another_unknown_git_command() - assert_in('test-option', exc_str(cme.exception)) - - # but other repos should not be affected: - gr2 = GitRepo(path2, create=True) - with assert_raises(GitCommandError) as cme: - gr2.repo.git.unknown_git_command() - assert_not_in('test-option', exc_str(cme.exception)) - - @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) def test_GitRepo_flyweight(path1, path2): @@ -1201,13 +1166,6 @@ def test_GitRepo_flyweight(path1, path2): ok_(repo1 == repo3) -@with_tempfile(mkdir=True) -@with_tempfile() -def test_GitRepo_flyweight_monitoring_inode(path, store): - # testing for issue #1512 - check_repo_deals_with_inode_change(GitRepo, path, store) - - @with_tree(tree={'ignore-sub.me': {'a_file.txt': 'some content'}, 'ignore.me': 'ignored content', 'dontigno.re': 'other content'}) @@ -1445,26 +1403,6 @@ def test_fake_dates(path): eq_(seconds_initial + 3, gr.get_commit_date()) -def test_guard_BadName(): - from gitdb.exc import BadName - - calls = [] - - class Vulnerable(object): - def precommit(self): - calls.append('precommit') - - @guard_BadName - def __call__(self, x, y=2): - if not calls: - calls.append(1) - raise BadName - return x+y - v = Vulnerable() - eq_(v(1, y=3), 4) - eq_(calls, [1, 'precommit']) - - @with_tree(tree={"foo": "foo content"}) def test_custom_runner_protocol(path): # Check that a runner with a non-default protocol gets wired up correctly. diff --git a/datalad/support/tests/utils.py b/datalad/support/tests/utils.py index 4008418e53..8b64cfa324 100644 --- a/datalad/support/tests/utils.py +++ b/datalad/support/tests/utils.py @@ -10,53 +10,3 @@ from datalad.support.external_versions import external_versions from datalad.tests.utils import * - - -def check_repo_deals_with_inode_change(class_, path, temp_store): - if external_versions["git"] == "3.0.0": - raise SkipTest("Fails due to GitPython regression (gh-3598)") - repo = class_(path, create=True) - with open(opj(path, "testfile.txt"), "w") as f: - f.write("whatever") - repo.add("testfile.txt") - repo.commit(msg="some load") - - # requesting HEAD info from - hexsha = repo.repo.head.object.hexsha - - # move everything to store - import os - import shutil - old_inode = os.stat(path).st_ino - shutil.copytree(path, temp_store, symlinks=True) - # kill original - # To make it "clean" we need to stop batched processes to not - # have anything holding that path (e.g. on windows) - # Unfortunately it is not enough ATM since GitPython also has - # cat-file --batched which we need to annihilate I guess - repo.precommit() - rmtree(path) - assert (not exists(path)) - # recreate - shutil.copytree(temp_store, path, symlinks=True) - new_inode = os.stat(path).st_ino - - if old_inode == new_inode: - raise SkipTest("inode did not change. Nothing to test for.") - - # Now, there is a running git process by GitPython's Repo instance, - # connected to an invalid inode! - # The *Repo needs to make sure to stop them, whenever we access the instance - # again (or request a flyweight instance). - - # The following two accesses fail in issue #1512: - # 1. requesting HEAD info from old instance - # - # Note: This checks that we don't hit a GitPython failure, so this test - # must use .repo.head.object.hexsha, not repo.get_hexsha(). - hexsha = repo.repo.head.object.hexsha - - # 2. get a "new" instance and requesting HEAD - repo2 = class_(path) - hexsha2 = repo2.repo.head.object.hexsha - diff --git a/datalad/tests/utils.py b/datalad/tests/utils.py index 898269c429..b21d4f6b23 100644 --- a/datalad/tests/utils.py +++ b/datalad/tests/utils.py @@ -340,7 +340,8 @@ def ok_clean_git(path, annex=None, head_modified=[], index_modified=[], eq_(sorted(r.untracked_files), sorted(untracked)) - repo = r.repo + import git + repo = git.Repo(r.path) if repo.index.entries.keys(): ok_(repo.head.is_valid()) From 31079c451c55b1c4f4d07fb22201822cd68cb1c4 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Fri, 21 Feb 2020 11:13:57 +0100 Subject: [PATCH 02/10] RF: Only inject GitPython into environment for tests Things like ok_clean_git() still use it. --- datalad/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/datalad/__init__.py b/datalad/__init__.py index c4d2591b91..74613e84be 100644 --- a/datalad/__init__.py +++ b/datalad/__init__.py @@ -43,14 +43,6 @@ # Other imports are interspersed with lgr.debug to ease troubleshooting startup # delays etc. -# If there is a bundled git, make sure GitPython uses it too: -from datalad.cmd import GitRunner -GitRunner._check_git_path() -if GitRunner._GIT_PATH: - import os - os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = \ - os.path.join(GitRunner._GIT_PATH, 'git') - from .config import ConfigManager cfg = ConfigManager() @@ -130,6 +122,15 @@ def setup_package(): """) _TEMP_PATHS_GENERATED.append(new_home) + # If there is a bundled git, make sure GitPython uses it too + # (some parts of the test utilities still rely on GitPython) + from datalad.cmd import GitRunner + GitRunner._check_git_path() + if GitRunner._GIT_PATH: + import os + os.environ['GIT_PYTHON_GIT_EXECUTABLE'] = \ + os.path.join(GitRunner._GIT_PATH, 'git') + # Re-load ConfigManager, since otherwise it won't consider global config # from new $HOME (see gh-4153 cfg.reload(force=True) From d7cf3609c3fd5a03fb1b27b0d70997cc4983e1fa Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Fri, 21 Feb 2020 11:38:16 +0100 Subject: [PATCH 03/10] RF: NO more exceptions imported from GitPython --- datalad/auto.py | 2 +- datalad/cmdline/helpers.py | 2 +- datalad/distribution/dataset.py | 10 ++++++---- datalad/plugin/check_dates.py | 7 ++++--- datalad/plugin/wtf.py | 6 ++++-- datalad/support/annexrepo.py | 5 ++--- datalad/support/exceptions.py | 12 ++++++++++++ datalad/support/gitrepo.py | 6 ++---- datalad/support/tests/test_gitrepo.py | 4 ++-- 9 files changed, 34 insertions(+), 20 deletions(-) diff --git a/datalad/auto.py b/datalad/auto.py index 8ccded9a43..d2fc35644a 100644 --- a/datalad/auto.py +++ b/datalad/auto.py @@ -23,7 +23,7 @@ from os.path import exists from os.path import isabs from os.path import join as opj -from git.exc import InvalidGitRepositoryError +from datalad.support.exceptions import InvalidGitRepositoryError from .utils import getpwd from .dochelpers import exc_str diff --git a/datalad/cmdline/helpers.py b/datalad/cmdline/helpers.py index 2912dd8c6b..9ab3a573a3 100644 --- a/datalad/cmdline/helpers.py +++ b/datalad/cmdline/helpers.py @@ -212,7 +212,7 @@ def get_repo_instance(path=curdir, class_=None): """ from os.path import ismount, exists, normpath, isabs - from git.exc import InvalidGitRepositoryError + from datalad.support.exceptions import InvalidGitRepositoryError from ..utils import expandpath from ..support.gitrepo import GitRepo from ..support.annexrepo import AnnexRepo diff --git a/datalad/distribution/dataset.py b/datalad/distribution/dataset.py index 71616a70e9..d192ae682c 100644 --- a/datalad/distribution/dataset.py +++ b/datalad/distribution/dataset.py @@ -29,15 +29,17 @@ # DueCredit from datalad.support.due import due from datalad.support.due_utils import duecredit_dataset -from datalad.support.exceptions import NoDatasetArgumentFound +from datalad.support.exceptions import ( + InvalidAnnexRepositoryError, + InvalidGitRepositoryError, + NoDatasetArgumentFound, + NoSuchPathError, +) from datalad.support.gitrepo import ( GitRepo, - InvalidGitRepositoryError, - NoSuchPathError ) from datalad.support.repo import PathBasedFlyweight from datalad.support.network import RI -from datalad.support.exceptions import InvalidAnnexRepositoryError from datalad.support import path as op import datalad.utils as ut diff --git a/datalad/plugin/check_dates.py b/datalad/plugin/check_dates.py index 746da18991..fbb9af83de 100644 --- a/datalad/plugin/check_dates.py +++ b/datalad/plugin/check_dates.py @@ -13,13 +13,14 @@ import os import time -from git.exc import InvalidGitRepositoryError - from datalad.dochelpers import exc_str from datalad.interface.base import Interface from datalad.interface.base import build_doc from datalad.interface.results import get_status_dict -from datalad.support.exceptions import MissingExternalDependency +from datalad.support.exceptions import ( + InvalidGitRepositoryError, + MissingExternalDependency, +) __docformat__ = "restructuredtext" diff --git a/datalad/plugin/wtf.py b/datalad/plugin/wtf.py index afdd14001b..6ce9243e07 100644 --- a/datalad/plugin/wtf.py +++ b/datalad/plugin/wtf.py @@ -26,8 +26,10 @@ ) from datalad.dochelpers import exc_str from datalad.support.external_versions import external_versions -from datalad.support.exceptions import CommandError -from datalad.support.gitrepo import InvalidGitRepositoryError +from datalad.support.exceptions import ( + CommandError, + InvalidGitRepositoryError, +) from datalad.version import __version__, __full_version__ lgr = logging.getLogger('datalad.plugin.wtf') diff --git a/datalad/support/annexrepo.py b/datalad/support/annexrepo.py index c75f48ad86..01218826b8 100644 --- a/datalad/support/annexrepo.py +++ b/datalad/support/annexrepo.py @@ -35,8 +35,6 @@ from multiprocessing import cpu_count from weakref import WeakValueDictionary -from git import InvalidGitRepositoryError - from datalad import ssh_manager from datalad.consts import ADJUSTED_BRANCH_EXPR from datalad.dochelpers import ( @@ -75,7 +73,6 @@ from .repo import RepoInterface from .gitrepo import ( GitRepo, - NoSuchPathError, _normalize_path, normalize_path, normalize_paths, @@ -95,10 +92,12 @@ BrokenExternalDependency, OutdatedExternalDependency, MissingExternalDependency, + NoSuchPathError, IncompleteResultsError, AccessDeniedError, AccessFailedError, InvalidAnnexRepositoryError, + InvalidGitRepositoryError, DirectModeNoLongerSupportedError ) diff --git a/datalad/support/exceptions.py b/datalad/support/exceptions.py index 1e249e56d9..78951db45a 100644 --- a/datalad/support/exceptions.py +++ b/datalad/support/exceptions.py @@ -223,6 +223,14 @@ class PathKnownToRepositoryError(Exception): pass +class GitError(Exception): + """ Base class for all package exceptions """ + + +class NoSuchPathError(GitError, OSError): + """ Thrown if a path could not be access by the system. """ + + class MissingBranchError(Exception): """Thrown if accessing a repository's branch, that is not available""" @@ -312,6 +320,10 @@ def __init__(self, id_, msg=None): self.msg = msg +class InvalidGitRepositoryError(GitError): + """ Thrown if the given repository appears to have an invalid format. """ + + class InvalidAnnexRepositoryError(RuntimeError): """Thrown if AnnexRepo was instantiated on a non-annex and without init=True""" diff --git a/datalad/support/gitrepo.py b/datalad/support/gitrepo.py index 44520ba837..7b4ef91f5d 100644 --- a/datalad/support/gitrepo.py +++ b/datalad/support/gitrepo.py @@ -43,10 +43,6 @@ from functools import wraps from weakref import WeakValueDictionary -from git.exc import ( - NoSuchPathError, - InvalidGitRepositoryError -) from datalad.log import log_progress from datalad.support.due import due, Doi @@ -91,6 +87,8 @@ FileNotInRepositoryError, GitIgnoreError, InvalidGitReferenceError, + InvalidGitRepositoryError, + NoSuchPathError, OutdatedExternalDependencyWarning, PathKnownToRepositoryError, ) diff --git a/datalad/support/tests/test_gitrepo.py b/datalad/support/tests/test_gitrepo.py index 2c761e0844..f3b0d03d5a 100644 --- a/datalad/support/tests/test_gitrepo.py +++ b/datalad/support/tests/test_gitrepo.py @@ -60,16 +60,16 @@ from datalad.support.gitrepo import ( _normalize_path, GitRepo, - InvalidGitRepositoryError, normalize_paths, - NoSuchPathError, to_options, ) from datalad.support.exceptions import ( CommandError, DeprecatedError, FileNotInRepositoryError, + InvalidGitRepositoryError, PathKnownToRepositoryError, + NoSuchPathError, ) from datalad.support.external_versions import external_versions from datalad.support.protocol import ExecutionTimeProtocol From 762526c152055ef487bc76141833dddec702f24a Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Fri, 21 Feb 2020 11:39:15 +0100 Subject: [PATCH 04/10] RF: Remove last non-test GitPython import (already unused) --- datalad/distribution/publish.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/datalad/distribution/publish.py b/datalad/distribution/publish.py index f569c4752b..e979aa7a13 100644 --- a/datalad/distribution/publish.py +++ b/datalad/distribution/publish.py @@ -15,8 +15,6 @@ from collections import OrderedDict from os.path import join as opj -from git.remote import PushInfo as PI - from datalad import ssh_manager from datalad.interface.annotate_paths import AnnotatePaths from datalad.interface.annotate_paths import annotated2content_by_ds From 46b86fb57e02ecef5837d9dafae672b478e71db1 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 25 Feb 2020 16:17:16 +0100 Subject: [PATCH 05/10] TST: Use annex.pidlock=true to overcome NFS issues --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8272dd813f..5e717fb000 100644 --- a/.travis.yml +++ b/.travis.yml @@ -232,7 +232,7 @@ install: # TMPDIRs - if [[ "${TMPDIR:-}" =~ .*/sym\ link ]]; then echo "Symlinking $TMPDIR"; ln -s /tmp "$TMPDIR"; fi - if [[ "${TMPDIR:-}" =~ .*/d\ i\ r ]]; then echo "mkdir $TMPDIR"; mkdir -p "$TMPDIR"; fi - - if [[ "${TMPDIR:-}" =~ .*/nfsmount ]]; then echo "mkdir $TMPDIR"; mkdir -p "$TMPDIR" "${TMPDIR}_"; echo "/tmp/nfsmount_ localhost(rw)" | sudo bash -c 'cat - > /etc/exports'; sudo apt-get install -y nfs-kernel-server; sudo exportfs -a; sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount; fi + - if [[ "${TMPDIR:-}" =~ .*/nfsmount ]]; then echo "mkdir $TMPDIR"; mkdir -p "$TMPDIR" "${TMPDIR}_"; echo "/tmp/nfsmount_ localhost(rw)" | sudo bash -c 'cat - > /etc/exports'; sudo apt-get install -y nfs-kernel-server; sudo exportfs -a; sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount; sudo git config --system annex.pidlock true ; fi # S3 - if [ ! -z "$UNSET_S3_SECRETS" ]; then echo "usetting"; unset DATALAD_datalad_test_s3_key_id DATALAD_datalad_test_s3_secret_id; fi # Install grunt to test run javascript frontend tests From e49e285f42091b4cc3cabc0b970ce796fde336c8 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Wed, 26 Feb 2020 07:44:23 +0100 Subject: [PATCH 06/10] RF: Wipe out ok_clean_git() Was replaced by assert_repo_status() a long time ago. With this change the last use of GitPython is gone. A shim for ok_clean_git() is kept that maps the most common use cases onto assert_repo_status() -- to be nice to extension devs. --- .../distribution/tests/test_create_sibling.py | 4 +- .../tests/test_create_test_dataset.py | 10 +- datalad/distribution/tests/test_dataset.py | 6 +- datalad/distribution/tests/test_get.py | 4 +- datalad/distribution/tests/test_install.py | 42 ++++---- datalad/distribution/tests/test_publish.py | 38 +++---- datalad/distribution/tests/test_uninstall.py | 62 ++++++------ datalad/distribution/tests/test_update.py | 21 ++-- .../tests/test_add_archive_content.py | 6 +- .../interface/tests/test_annotate_paths.py | 12 +-- datalad/interface/tests/test_diff.py | 10 +- datalad/interface/tests/test_utils.py | 12 +-- .../metadata/extractors/tests/test_audio.py | 4 +- .../metadata/extractors/tests/test_base.py | 4 +- .../metadata/extractors/tests/test_exif.py | 4 +- .../metadata/extractors/tests/test_image.py | 4 +- datalad/metadata/extractors/tests/test_xmp.py | 4 +- datalad/metadata/tests/test_aggregation.py | 36 +++---- datalad/metadata/tests/test_base.py | 6 +- .../metadata/tests/test_extract_metadata.py | 4 +- datalad/metadata/tests/test_search.py | 4 +- datalad/plugin/tests/test_plugins.py | 8 +- datalad/support/tests/test_annexrepo.py | 40 ++++---- datalad/support/tests/test_gitrepo.py | 24 ++--- datalad/tests/test_utils_testrepos.py | 6 +- datalad/tests/utils.py | 99 +++---------------- 26 files changed, 199 insertions(+), 275 deletions(-) diff --git a/datalad/distribution/tests/test_create_sibling.py b/datalad/distribution/tests/test_create_sibling.py index 0018440924..36fd49d204 100644 --- a/datalad/distribution/tests/test_create_sibling.py +++ b/datalad/distribution/tests/test_create_sibling.py @@ -33,13 +33,13 @@ assert_not_equal, assert_not_in, assert_raises, + assert_repo_status, assert_result_count, assert_status, create_tree, eq_, get_mtimes_and_digests, ok_, - ok_clean_git, ok_endswith, ok_exists, ok_file_has_content, @@ -440,7 +440,7 @@ def check_target_ssh_since(use_ssh, origin, src_path, target_path): # get a new subdataset and make sure it is committed in the super source.create('brandnew') eq_(len(source.subdatasets()), 3) - ok_clean_git(source.path) + assert_repo_status(source.path) # and now we create a sibling for the new subdataset only assert_create_sshwebserver( diff --git a/datalad/distribution/tests/test_create_test_dataset.py b/datalad/distribution/tests/test_create_test_dataset.py index 0d0a6d6f28..5bfdcfbf4c 100644 --- a/datalad/distribution/tests/test_create_test_dataset.py +++ b/datalad/distribution/tests/test_create_test_dataset.py @@ -15,8 +15,8 @@ from datalad.tests.utils import ( with_tempfile, assert_raises, + assert_repo_status, ok_, - ok_clean_git, known_failure_githubci_win, ) from datalad.utils import ( @@ -49,7 +49,7 @@ def test_create_test_dataset(): dss = create_test_dataset(spec='2/1-2') ok_(5 <= len(dss) <= 7) # at least five - 1 top, two on top level, 1 in each for ds in dss: - ok_clean_git(ds, annex=None) # some of them are annex but we just don't check + assert_repo_status(ds, annex=None) # some of them are annex but we just don't check ok_(len(glob(opj(ds, 'file*')))) @@ -59,7 +59,7 @@ def test_create_1test_dataset(): with swallow_outputs(): dss = create_test_dataset() eq_(len(dss), 1) - ok_clean_git(dss[0], annex=False) + assert_repo_status(dss[0], annex=False) @with_tempfile(mkdir=True) @@ -70,7 +70,7 @@ def test_new_relpath(topdir): eq_(dss[0], opj(topdir, 'testds')) eq_(len(dss), 2) # 1 top + 1 sub-dataset as demanded for ds in dss: - ok_clean_git(ds, annex=False) + assert_repo_status(ds, annex=False) @known_failure_githubci_win @@ -84,7 +84,7 @@ def test_hierarchy(topdir): eq_(len(dss), 3) eq_(dss[0], topdir) for ids, ds in enumerate(dss): - ok_clean_git(ds, annex=False) + assert_repo_status(ds, annex=False) # each one should have 2 commits (but the last one)-- one for file and # another one for sub-dataset repo = GitRepo(ds) diff --git a/datalad/distribution/tests/test_dataset.py b/datalad/distribution/tests/test_dataset.py index 4511ca4c89..6ac4817441 100644 --- a/datalad/distribution/tests/test_dataset.py +++ b/datalad/distribution/tests/test_dataset.py @@ -48,13 +48,13 @@ assert_is_not_none, assert_not_equal, assert_raises, + assert_repo_status, assert_result_count, assert_true, eq_, known_failure_windows, OBSCURE_FILENAME, ok_, - ok_clean_git, SkipTest, with_tempfile, with_testrepos, @@ -385,7 +385,7 @@ def test_property_reevaluation(repo1): assert_is_none(ds.id) ds.create() - ok_clean_git(repo1) + assert_repo_status(repo1) # after creation, we have `repo`, and `config` was reevaluated to point # to the repo's config: assert_is_not_none(ds.repo) @@ -409,7 +409,7 @@ def test_property_reevaluation(repo1): assert_is_none(ds.id) ds.create() - ok_clean_git(repo1) + assert_repo_status(repo1) # after recreation everything is sane again: assert_is_not_none(ds.repo) assert_is_not_none(ds.config) diff --git a/datalad/distribution/tests/test_get.py b/datalad/distribution/tests/test_get.py index 305ce8ae0f..6930bd66dc 100644 --- a/datalad/distribution/tests/test_get.py +++ b/datalad/distribution/tests/test_get.py @@ -30,7 +30,6 @@ ) from datalad.tests.utils import ( ok_, - ok_clean_git, eq_, with_tempfile, with_testrepos, @@ -41,6 +40,7 @@ assert_status, assert_in_results, assert_not_in_results, + assert_repo_status, assert_result_count, assert_message, serve_path_via_http, @@ -334,7 +334,7 @@ def test_get_recurse_subdatasets(src, path): ok_(subds1.repo.file_has_content('test-annex.dat') is False) ok_(subds2.repo.file_has_content('test-annex.dat') is False) - ok_clean_git(subds1.path) + assert_repo_status(subds1.path) # explicitly given path in subdataset => implicit recursion: # MIH: Nope, we fulfill the dataset handle, but that doesn't # imply fulfilling all file handles diff --git a/datalad/distribution/tests/test_install.py b/datalad/distribution/tests/test_install.py index 0d66863429..c94760494d 100644 --- a/datalad/distribution/tests/test_install.py +++ b/datalad/distribution/tests/test_install.py @@ -57,11 +57,11 @@ assert_not_in, assert_raises, assert_is_instance, + assert_repo_status, assert_result_count, assert_status, assert_in_results, ok_startswith, - ok_clean_git, serve_path_via_http, swallow_logs, use_cassette, @@ -103,7 +103,7 @@ def _test_guess_dot_git(annex, path, url, tdir): assert_not_in("Failed to get annex.uuid", cml.out) eq_(realpath(installed.path), realpath(tdir)) ok_(exists(tdir)) - ok_clean_git(tdir, annex=annex) + assert_repo_status(tdir, annex=annex) def test_guess_dot_git(): @@ -217,14 +217,14 @@ def test_install_simple_local(src, path): ok_(GitRepo.is_valid_repo(ds.path)) eq_(set(ds.repo.get_indexed_files()), {'test.dat', 'INFO.txt'}) - ok_clean_git(path, annex=False) + assert_repo_status(path, annex=False) else: # must be an annex ok_(isinstance(ds.repo, AnnexRepo)) ok_(AnnexRepo.is_valid_repo(ds.path, allow_noninitialized=False)) eq_(set(ds.repo.get_indexed_files()), {'test.dat', 'INFO.txt', 'test-annex.dat'}) - ok_clean_git(path, annex=True) + assert_repo_status(path, annex=True) # no content was installed: ok_(not ds.repo.file_has_content('test-annex.dat')) uuid_before = ds.repo.uuid @@ -249,7 +249,7 @@ def test_install_dataset_from_just_source(url, path): ok_startswith(ds.path, path) ok_(ds.is_installed()) ok_(GitRepo.is_valid_repo(ds.path)) - ok_clean_git(ds.path, annex=None) + assert_repo_status(ds.path, annex=None) assert_in('INFO.txt', ds.repo.get_indexed_files()) @@ -263,7 +263,7 @@ def test_install_dataset_from_instance(src, dst): ok_startswith(clone.path, dst) ok_(clone.is_installed()) ok_(GitRepo.is_valid_repo(clone.path)) - ok_clean_git(clone.path, annex=None) + assert_repo_status(clone.path, annex=None) assert_in('INFO.txt', clone.repo.get_indexed_files()) @@ -280,7 +280,7 @@ def test_install_dataset_from_just_source_via_path(url, path): ok_startswith(ds.path, path) ok_(ds.is_installed()) ok_(GitRepo.is_valid_repo(ds.path)) - ok_clean_git(ds.path, annex=None) + assert_repo_status(ds.path, annex=None) assert_in('INFO.txt', ds.repo.get_indexed_files()) @@ -301,7 +301,7 @@ def test_install_dataladri(src, topurl, path): swallow_logs(): ds = install(path, source='///ds') eq_(ds.path, path) - ok_clean_git(path, annex=False) + assert_repo_status(path, annex=False) ok_file_has_content(opj(path, 'test.txt'), 'some') @@ -399,35 +399,35 @@ def test_install_recursive_with_data(src, path): def test_install_into_dataset(source, top_path): ds = create(top_path) - ok_clean_git(ds.path) + assert_repo_status(ds.path) subds = ds.install("sub", source=source, save=False) ok_(isdir(opj(subds.path, '.git'))) ok_(subds.is_installed()) assert_in('sub', ds.subdatasets(result_xfm='relpaths')) # sub is clean: - ok_clean_git(subds.path, annex=None) + assert_repo_status(subds.path, annex=None) # top is too: - ok_clean_git(ds.path, annex=None) + assert_repo_status(ds.path, annex=None) ds.save(message='addsub') # now it is: - ok_clean_git(ds.path, annex=None) + assert_repo_status(ds.path, annex=None) # but we could also save while installing and there should be no side-effect # of saving any other changes if we state to not auto-save changes # Create a dummy change create_tree(ds.path, {'dummy.txt': 'buga'}) - ok_clean_git(ds.path, untracked=['dummy.txt']) + assert_repo_status(ds.path, untracked=['dummy.txt']) subds_ = ds.install("sub2", source=source) eq_(subds_.path, opj(ds.path, "sub2")) # for paranoid yoh ;) - ok_clean_git(ds.path, untracked=['dummy.txt']) + assert_repo_status(ds.path, untracked=['dummy.txt']) # and we should achieve the same behavior if we create a dataset # and then decide to add it create(_path_(top_path, 'sub3')) - ok_clean_git(ds.path, untracked=['dummy.txt', 'sub3/']) + assert_repo_status(ds.path, untracked=['dummy.txt', 'sub3/']) ds.save('sub3') - ok_clean_git(ds.path, untracked=['dummy.txt']) + assert_repo_status(ds.path, untracked=['dummy.txt']) @known_failure_windows #FIXME @@ -440,7 +440,7 @@ def test_failed_install_multiple(top_path): create(_path_(top_path, 'ds1')) create(_path_(top_path, 'ds3')) - ok_clean_git(ds.path, annex=None, untracked=['ds1/', 'ds3/']) + assert_repo_status(ds.path, annex=None, untracked=['ds1/', 'ds3/']) # specify install with multiple paths and one non-existing with assert_raises(IncompleteResultsError) as cme: @@ -448,9 +448,9 @@ def test_failed_install_multiple(top_path): on_failure='continue') # install doesn't add existing submodules -- add does that - ok_clean_git(ds.path, annex=None, untracked=['ds1/', 'ds3/']) + assert_repo_status(ds.path, annex=None, untracked=['ds1/', 'ds3/']) ds.save(['ds1', 'ds3']) - ok_clean_git(ds.path, annex=None) + assert_repo_status(ds.path, annex=None) # those which succeeded should be saved now eq_(ds.subdatasets(result_xfm='relpaths'), ['crcns', 'ds1', 'ds3']) # and those which didn't -- listed @@ -610,14 +610,14 @@ def test_reckless(path, top_path): } }) @with_tempfile(mkdir=True) -@skip_if_on_windows # Due to "another process error" and buggy ok_clean_git +@skip_if_on_windows # Due to "another process error" def test_install_recursive_repeat(src, path): top_src = Dataset(src).create(force=True) sub1_src = top_src.create('sub 1', force=True) sub2_src = top_src.create('sub 2', force=True) subsub_src = sub1_src.create('subsub', force=True) top_src.save(recursive=True) - ok_clean_git(top_src.path) + assert_repo_status(top_src.path) # install top level: top_ds = install(path, source=src) diff --git a/datalad/distribution/tests/test_publish.py b/datalad/distribution/tests/test_publish.py index b37f24172b..012d9ea204 100644 --- a/datalad/distribution/tests/test_publish.py +++ b/datalad/distribution/tests/test_publish.py @@ -39,6 +39,7 @@ assert_not_equal, assert_not_in, assert_raises, + assert_repo_status, assert_result_count, assert_status, create_tree, @@ -46,7 +47,6 @@ known_failure_windows, neq_, ok_, - ok_clean_git, ok_file_has_content, serve_path_via_http, skip_if_on_windows, @@ -139,8 +139,8 @@ def test_publish_simple(origin, src_path, dst_path): res = publish(dataset=source, to="target", result_xfm='datasets') eq_(res, [source]) - ok_clean_git(source.repo, annex=None) - ok_clean_git(target, annex=None) + assert_repo_status(source.repo, annex=None) + assert_repo_status(target, annex=None) eq_(list(target.get_branch_commits_("master")), list(source.repo.get_branch_commits_("master"))) @@ -149,8 +149,8 @@ def test_publish_simple(origin, src_path, dst_path): # and nothing is pushed assert_result_count(res, 1, status='notneeded') - ok_clean_git(source.repo, annex=None) - ok_clean_git(target, annex=None) + assert_repo_status(source.repo, annex=None) + assert_repo_status(target, annex=None) eq_(list(target.get_branch_commits_("master")), list(source.repo.get_branch_commits_("master"))) eq_(list(target.get_branch_commits_("git-annex")), @@ -165,12 +165,12 @@ def test_publish_simple(origin, src_path, dst_path): f.write("Some additional stuff.") source.save(opj(src_path, 'test_mod_file'), to_git=True, message="Modified.") - ok_clean_git(source.repo, annex=None) + assert_repo_status(source.repo, annex=None) res = publish(dataset=source, to='target', result_xfm='datasets') eq_(res, [source]) - ok_clean_git(dst_path, annex=None) + assert_repo_status(dst_path, annex=None) eq_(list(target.get_branch_commits_("master")), list(source.repo.get_branch_commits_("master"))) # Since git-annex 6.20170220, post-receive hook gets triggered @@ -205,8 +205,8 @@ def test_publish_plain_git(origin, src_path, dst_path): res = publish(dataset=source, to="target", result_xfm='datasets') eq_(res, [source]) - ok_clean_git(source.repo, annex=None) - ok_clean_git(target, annex=None) + assert_repo_status(source.repo, annex=None) + assert_repo_status(target, annex=None) eq_(list(target.get_branch_commits_("master")), list(source.repo.get_branch_commits_("master"))) @@ -215,8 +215,8 @@ def test_publish_plain_git(origin, src_path, dst_path): # and nothing is pushed assert_result_count(res, 1, status='notneeded') - ok_clean_git(source.repo, annex=None) - ok_clean_git(target, annex=None) + assert_repo_status(source.repo, annex=None) + assert_repo_status(target, annex=None) eq_(list(target.get_branch_commits_("master")), list(source.repo.get_branch_commits_("master"))) @@ -225,12 +225,12 @@ def test_publish_plain_git(origin, src_path, dst_path): f.write("Some additional stuff.") source.save(opj(src_path, 'test_mod_file'), to_git=True, message="Modified.") - ok_clean_git(source.repo, annex=None) + assert_repo_status(source.repo, annex=None) res = publish(dataset=source, to='target', result_xfm='datasets') eq_(res, [source]) - ok_clean_git(dst_path, annex=None) + assert_repo_status(dst_path, annex=None) eq_(list(target.get_branch_commits_("master")), list(source.repo.get_branch_commits_("master"))) @@ -570,11 +570,11 @@ def test_publish_depends( source.create_sibling( 'ssh://datalad-test' + target3_path, name='target3') - ok_clean_git(src_path) + assert_repo_status(src_path) # introduce change in source create_tree(src_path, {'probe1': 'probe1'}) source.save('probe1') - ok_clean_git(src_path) + assert_repo_status(src_path) # only the source has the probe ok_file_has_content(opj(src_path, 'probe1'), 'probe1') for p in (target1_path, target2_path, target3_path): @@ -611,14 +611,14 @@ def test_gh1426(origin_path, target_path): 'receive.denyCurrentBranch', 'updateInstead', where='local') origin.siblings('add', name='target', url=target_path) origin.publish(to='target') - ok_clean_git(origin.path) - ok_clean_git(target.path) + assert_repo_status(origin.path) + assert_repo_status(target.path) eq_(origin.repo.get_hexsha(), target.get_hexsha()) # gist of #1426 is that a newly added subdataset does not cause the # superdataset to get published origin.create('sub') - ok_clean_git(origin.path) + assert_repo_status(origin.path) assert_not_equal(origin.repo.get_hexsha(), target.get_hexsha()) # now push res = origin.publish(to='target') @@ -642,7 +642,7 @@ def test_publish_gh1691(origin, src_path, dst_path): # some content modification of the superdataset create_tree(src_path, {'probe1': 'probe1'}) source.save('probe1') - ok_clean_git(src_path) + assert_repo_status(src_path) # create the target(s): source.create_sibling( diff --git a/datalad/distribution/tests/test_uninstall.py b/datalad/distribution/tests/test_uninstall.py index c024ee6acc..08d2727a30 100644 --- a/datalad/distribution/tests/test_uninstall.py +++ b/datalad/distribution/tests/test_uninstall.py @@ -36,10 +36,10 @@ assert_raises, assert_status, assert_in, + assert_repo_status, assert_result_count, assert_result_values_cond, ok_file_under_git, - ok_clean_git, with_tempfile, with_tree, create_tree, @@ -92,7 +92,7 @@ def test_clean_subds_removal(path): subds1 = ds.create('one') subds2 = ds.create('two') eq_(sorted(ds.subdatasets(result_xfm='relpaths')), ['one', 'two']) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # now kill one res = ds.remove('one', result_xfm=None) # subds1 got uninstalled, and ds got the removal of subds1 saved @@ -100,7 +100,7 @@ def test_clean_subds_removal(path): assert_result_count(res, 1, path=subds1.path, action='remove', status='ok') assert_result_count(res, 1, path=ds.path, action='save', status='ok') ok_(not subds1.is_installed()) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # two must remain eq_(ds.subdatasets(result_xfm='relpaths'), ['two']) # one is gone @@ -109,12 +109,12 @@ def test_clean_subds_removal(path): ds.create('three') eq_(sorted(ds.subdatasets(result_xfm='relpaths')), ['three', 'two']) ds.uninstall('two') - ok_clean_git(ds.path) + assert_repo_status(ds.path) eq_(sorted(ds.subdatasets(result_xfm='relpaths')), ['three', 'two']) ok_(not subds2.is_installed()) assert(exists(subds2.path)) res = ds.remove('two', result_xfm='datasets') - ok_clean_git(ds.path) + assert_repo_status(ds.path) # subds2 was already uninstalled, now ds got the removal of subds2 saved assert(not exists(subds2.path)) eq_(ds.subdatasets(result_xfm='relpaths'), ['three']) @@ -253,24 +253,24 @@ def test_uninstall_multiple_paths(path): ds = Dataset(path).create(force=True) subds = ds.create('deep', force=True) subds.save(recursive=True) - ok_clean_git(subds.path) + assert_repo_status(subds.path) # needs to be able to add a combination of staged files, modified submodule, # and untracked files ds.save(recursive=True) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # drop content of all 'kill' files topfile = 'kill' deepfile = opj('deep', 'dir', 'kill') # use a tuple not a list! should also work ds.drop((topfile, deepfile), check=False) - ok_clean_git(ds.path) + assert_repo_status(ds.path) files_left = glob(opj(ds.path, '*', '*', '*')) + glob(opj(ds.path, '*')) ok_(all([f.endswith('keep') for f in files_left if exists(f) and not isdir(f)])) ok_(not ds.repo.file_has_content(topfile)) ok_(not subds.repo.file_has_content(opj(*psplit(deepfile)[1:]))) # remove handles for all 'kill' files ds.remove([topfile, deepfile], check=False) - ok_clean_git(ds.path) + assert_repo_status(ds.path) files_left = glob(opj(ds.path, '*', '*', '*')) + glob(opj(ds.path, '*')) ok_(all([f.endswith('keep') for f in files_left if exists(f) and not isdir(f)])) ok_(not any([f.endswith(topfile) for f in files_left])) @@ -282,12 +282,12 @@ def test_uninstall_dataset(path): ok_(not ds.is_installed()) ds.create() ok_(ds.is_installed()) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # would only drop data ds.drop() # actually same as this, for cmdline compat reasons ds.drop(path=[]) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # removing entire dataset, uninstall will refuse to act on top-level # datasets assert_raises(IncompleteResultsError, ds.uninstall) @@ -302,7 +302,7 @@ def test_uninstall_dataset(path): def test_remove_file_handle_only(path): ds = Dataset(path).create(force=True) ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) # make sure there is any key ok_(len(ds.repo.get_file_key('one'))) # both files link to the same key @@ -339,8 +339,8 @@ def test_uninstall_recursive(path): assert_result_count(res, 1, action='save', status='ok', type='dataset') # save all -> all clean ds.save(recursive=True) - ok_clean_git(subds.path) - ok_clean_git(ds.path) + assert_repo_status(subds.path) + assert_repo_status(ds.path) # now uninstall in subdataset through superdataset target_fname = opj('deep', 'dir', 'test') # sane starting point @@ -360,24 +360,24 @@ def test_uninstall_recursive(path): lname = opj(ds.path, target_fname) ok_(not exists(lname)) # entire hierarchy saved - ok_clean_git(subds.path) - ok_clean_git(ds.path) + assert_repo_status(subds.path) + assert_repo_status(ds.path) # now same with actual handle removal # content is dropped already, so no checks in place anyway ds.remove(target_fname, check=True, recursive=True) ok_(not exists(lname) and not lexists(lname)) - ok_clean_git(subds.path) - ok_clean_git(ds.path) + assert_repo_status(subds.path) + assert_repo_status(ds.path) @with_tempfile() def test_remove_dataset_hierarchy(path): ds = Dataset(path).create() ds.create('deep') - ok_clean_git(ds.path) + assert_repo_status(ds.path) # fail on missing --recursive because subdataset is present assert_raises(IncompleteResultsError, ds.remove) - ok_clean_git(ds.path) + assert_repo_status(ds.path) ds.remove(recursive=True) # completely gone ok_(not ds.is_installed()) @@ -385,7 +385,7 @@ def test_remove_dataset_hierarchy(path): # now do it again, but without a reference dataset ds = Dataset(path).create() ds.create('deep') - ok_clean_git(ds.path) + assert_repo_status(ds.path) remove(dataset=ds.path, recursive=True) # completely gone ok_(not ds.is_installed()) @@ -399,13 +399,13 @@ def test_careless_subdataset_uninstall(path): subds1 = ds.create('deep1') ds.create('deep2') eq_(sorted(ds.subdatasets(result_xfm='relpaths')), ['deep1', 'deep2']) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # now we kill the sub without the parent knowing subds1.uninstall() ok_(not subds1.is_installed()) # mountpoint exists ok_(exists(subds1.path)) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # parent still knows the sub eq_(sorted(ds.subdatasets(result_xfm='relpaths')), ['deep1', 'deep2']) @@ -420,7 +420,7 @@ def test_kill(path): ds.save("file.dat") subds = ds.create('deep1') eq_(sorted(ds.subdatasets(result_xfm='relpaths')), ['deep1']) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # and we fail to remove since content can't be dropped res = ds.remove(on_failure='ignore') @@ -451,7 +451,7 @@ def test_remove_recreation(path): ds = create(path) ds.remove() ds = create(path) - ok_clean_git(ds.path) + assert_repo_status(ds.path) ok_(ds.is_installed()) @@ -502,7 +502,7 @@ def test_failon_nodrop(path): sub = ds.create('sub') create_tree(sub.path, {'test': 'content'}) ds.save(opj('sub', 'test')) - ok_clean_git(ds.path) + assert_repo_status(ds.path) eq_(['test'], sub.repo.get_annexed_files(with_content_only=True)) # we put one file into the dataset's annex, no redundant copies # neither uninstall nor remove should work @@ -521,9 +521,9 @@ def test_uninstall_without_super(path): # is just placed underneath the parent, but not an actual subdataset parent = Dataset(path).create() sub = parent.create('sub') - ok_clean_git(parent.path) + assert_repo_status(parent.path) nosub = create(opj(parent.path, 'nosub')) - ok_clean_git(nosub.path) + assert_repo_status(nosub.path) subreport = parent.subdatasets() assert_result_count(subreport, 1, path=sub.path) assert_result_count(subreport, 0, path=nosub.path) @@ -551,7 +551,7 @@ def test_drop_nocrash_absent_subds(path): parent = Dataset(path).create() sub = parent.create('sub') parent.uninstall('sub') - ok_clean_git(parent.path) + assert_repo_status(parent.path) with chpwd(path): assert_status('notneeded', drop('.', recursive=True)) @@ -560,7 +560,7 @@ def test_drop_nocrash_absent_subds(path): def test_remove_more_than_one(path): ds = Dataset(path).create(force=True) ds.save() - ok_clean_git(path) + assert_repo_status(path) # ensure #1912 stays resolved ds.remove(['one', 'two'], check=False) - ok_clean_git(path) + assert_repo_status(path) diff --git a/datalad/distribution/tests/test_update.py b/datalad/distribution/tests/test_update.py index 954d6875f4..2b7a5a1a56 100644 --- a/datalad/distribution/tests/test_update.py +++ b/datalad/distribution/tests/test_update.py @@ -42,9 +42,8 @@ ok_, create_tree, ok_file_has_content, - ok_clean_git, - assert_repo_status, assert_status, + assert_repo_status, assert_result_count, assert_in_results, SkipTest, @@ -78,18 +77,18 @@ def test_update_simple(origin, src_path, dst_path): dest = install(dst_path, source=src_path, recursive=True) # test setup done; # assert all fine - ok_clean_git(dst_path) - ok_clean_git(src_path) + assert_repo_status(dst_path) + assert_repo_status(src_path) # update yields nothing => up-to-date assert_status('ok', dest.update()) - ok_clean_git(dst_path) + assert_repo_status(dst_path) # modify origin: with open(opj(src_path, "update.txt"), "w") as f: f.write("Additional content") source.save(path="update.txt", message="Added update.txt") - ok_clean_git(src_path) + assert_repo_status(src_path) # update without `merge` only fetches: assert_status('ok', dest.update()) @@ -363,12 +362,12 @@ def test_update_volatile_subds(originpath, otherpath, destpath): # modify ds and subds at origin create_tree(origin.path, {'mike': 'this', sname: {'probe': 'little'}}) origin.save(recursive=True) - ok_clean_git(origin.path) + assert_repo_status(origin.path) # updates for both datasets should come down the pipe assert_result_count(ds.update(merge=True, recursive=True), 2, action='update', status='ok', type='dataset') - ok_clean_git(ds.path) + assert_repo_status(ds.path) # now remove just-installed subdataset from origin again origin.remove(sname, check=False) @@ -386,7 +385,7 @@ def test_update_volatile_subds(originpath, otherpath, destpath): # not using a bound method, not giving a parentds, should # not be needed to get a clean dataset remove(op.join(ds.path, sname), check=False) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # new separate subdataset, not within the origin dataset otherds = Dataset(otherpath).create() @@ -394,13 +393,13 @@ def test_update_volatile_subds(originpath, otherpath, destpath): ds.install(source=otherds.path, path='other') create_tree(otherds.path, {'brand': 'new'}) otherds.save() - ok_clean_git(otherds.path) + assert_repo_status(otherds.path) # pull in changes res = ds.update(merge=True, recursive=True) assert_result_count( res, 2, status='ok', action='update', type='dataset') # the next is to check for #2858 - ok_clean_git(ds.path) + assert_repo_status(ds.path) @known_failure_windows #FIXME diff --git a/datalad/interface/tests/test_add_archive_content.py b/datalad/interface/tests/test_add_archive_content.py index 97b347d76b..b232fbe9ae 100644 --- a/datalad/interface/tests/test_add_archive_content.py +++ b/datalad/interface/tests/test_add_archive_content.py @@ -31,6 +31,7 @@ assert_not_in, assert_raises, assert_re_in, + assert_repo_status, assert_result_values_cond, assert_true, create_tree, @@ -39,7 +40,6 @@ known_failure_githubci_win, ok_, ok_archives_caches, - ok_clean_git, ok_file_under_git, serve_path_via_http, slow, @@ -510,7 +510,7 @@ def test_add_delete_after_and_drop_subdir(self): opj('subdir', '1.tar'), delete_after=True, drop_after=True) - ok_clean_git(self.annex.path) + assert_repo_status(self.annex.path) commits_after_master = list(self.annex.get_branch_commits_()) commits_after = list(self.annex.get_branch_commits_('git-annex')) # There should be a single commit for all additions +1 to @@ -533,7 +533,7 @@ def test_add_delete_after_and_drop_subdir(self): delete_after=True, drop_after=True, allow_dirty=True) - ok_clean_git(self.annex.path, untracked=['dummy.txt']) + assert_repo_status(self.annex.path, untracked=['dummy.txt']) assert_equal(len(list(self.annex.get_branch_commits_())), len(commits_prior_master)) diff --git a/datalad/interface/tests/test_annotate_paths.py b/datalad/interface/tests/test_annotate_paths.py index cefb3aa4ec..3225edda93 100644 --- a/datalad/interface/tests/test_annotate_paths.py +++ b/datalad/interface/tests/test_annotate_paths.py @@ -25,8 +25,8 @@ from datalad.tests.utils import ( with_tree, with_tempfile, - ok_clean_git, eq_, + assert_repo_status, assert_result_count, assert_raises, assert_not_in, @@ -89,7 +89,7 @@ def test_annotate_paths(dspath, nodspath): # this test doesn't use API`remove` to avoid circularities ds = make_demo_hierarchy_datasets(dspath, demo_hierarchy) ds.save(recursive=True) - ok_clean_git(ds.path) + assert_repo_status(ds.path) with chpwd(dspath): # with and without an explicitly given path the result is almost the @@ -243,7 +243,7 @@ def test_get_modified_subpaths(path): subb = ds.create('bb', force=True) subsub = ds.create(opj('bb', 'bba', 'bbaa'), force=True) ds.save(recursive=True) - ok_clean_git(path) + assert_repo_status(path) orig_base_commit = ds.repo.get_hexsha() @@ -298,7 +298,7 @@ def test_get_modified_subpaths(path): # add/save everything, become clean ds.save(recursive=True) - ok_clean_git(path) + assert_repo_status(path) # nothing is reported as modified assert_result_count( get_modified_subpaths( @@ -330,7 +330,7 @@ def test_get_modified_subpaths(path): # deal with removal (force insufiicient copies error) ds.remove(suba.path, check=False) - ok_clean_git(path) + assert_repo_status(path) res = list(get_modified_subpaths([dict(path=ds.path)], ds, 'HEAD~1..HEAD')) # removed submodule + .gitmodules update assert_result_count(res, 2) @@ -352,7 +352,7 @@ def test_recurseinto(dspath, dest): set_property=[('datalad-recursiveinstall', 'skip')]) assert_result_count(res, 1, path=opj(ds.path, 'b', 'bb')) ds.save('b', recursive=True) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # recursive install, should skip the entire bb branch res = install(source=ds.path, path=dest, recursive=True, diff --git a/datalad/interface/tests/test_diff.py b/datalad/interface/tests/test_diff.py index cdca4160be..dc2da60d5e 100644 --- a/datalad/interface/tests/test_diff.py +++ b/datalad/interface/tests/test_diff.py @@ -23,11 +23,11 @@ known_failure_windows, with_tempfile, with_tree, - ok_clean_git, create_tree, ok_, eq_, assert_status, + assert_repo_status, assert_result_count, known_failure_githubci_win, ) @@ -49,7 +49,7 @@ def test_magic_number(): @with_tempfile(mkdir=True) def test_diff(path, norepo): ds = Dataset(path).create() - ok_clean_git(ds.path) + assert_repo_status(ds.path) # reports stupid revision input assert_result_count( ds._diff(revision='WTF', on_failure='ignore'), @@ -65,7 +65,7 @@ def test_diff(path, norepo): # let's introduce a known change create_tree(ds.path, {'new': 'empty'}) ds.save(to_git=True) - ok_clean_git(ds.path) + assert_repo_status(ds.path) res = ds._diff(revision='HEAD~1') assert_result_count(res, 1) assert_result_count( @@ -98,7 +98,7 @@ def test_diff(path, norepo): ds._diff(revision='HEAD'), 1, action='diff', path=opj(ds.path, 'new'), state='modified') ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) # untracked stuff create_tree(ds.path, {'deep': {'down': 'untracked', 'down2': 'tobeadded'}}) @@ -163,7 +163,7 @@ def test_diff_recursive(path): ds.save(sub.path) # save addition in parent ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) # look at the last change, only one file was added res = ds._diff(revision='HEAD~1..HEAD') assert_result_count(res, 1) diff --git a/datalad/interface/tests/test_utils.py b/datalad/interface/tests/test_utils.py index 372b56f640..a754dc1a53 100644 --- a/datalad/interface/tests/test_utils.py +++ b/datalad/interface/tests/test_utils.py @@ -23,9 +23,9 @@ assert_not_equal, assert_not_in, assert_raises, + assert_repo_status, assert_true, ok_, - ok_clean_git, slow, with_tempfile, with_tree, @@ -103,7 +103,7 @@ def test_dirty(path): # not added to super on purpose! subds = ds.create('subds') _check_all_clean(subds, subds.repo.get_hexsha()) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # subdataset must be added as a submodule! assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds']) @@ -150,7 +150,7 @@ def test_save_hierarchy(path): # this test doesn't use API`remove` to avoid circularities ds = make_demo_hierarchy_datasets(path, demo_hierarchy) ds.save(recursive=True) - ok_clean_git(ds.path) + assert_repo_status(ds.path) ds_bb = Dataset(opj(ds.path, 'b', 'bb')) ds_bba = Dataset(opj(ds_bb.path, 'bba')) ds_bbaa = Dataset(opj(ds_bba.path, 'bbaa')) @@ -164,7 +164,7 @@ def test_save_hierarchy(path): # it has saved all changes in the subtrees spanned # by the given datasets, but nothing else for d in (ds_bb, ds_bba, ds_bbaa): - ok_clean_git(d.path) + assert_repo_status(d.path) ok_(ds.repo.dirty) # now with two modified repos d = Dataset(opj(ds.path, 'd')) @@ -175,7 +175,7 @@ def test_save_hierarchy(path): # generator d.save(recursive=True) for d in (d, da, db): - ok_clean_git(d.path) + assert_repo_status(d.path) ok_(ds.repo.dirty) # and now with files all over the place and saving # all the way to the root @@ -328,7 +328,7 @@ def test_discover_ds_trace(path, otherdir): # subject is also involved in this assert_true(exists(opj(db, 'file_db'))) ds.save(recursive=True) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # now two datasets which are not available locally, but we # know about them (e.g. from metadata) dba = opj(db, 'sub', 'dba') diff --git a/datalad/metadata/extractors/tests/test_audio.py b/datalad/metadata/extractors/tests/test_audio.py index 099789654e..c54f566970 100644 --- a/datalad/metadata/extractors/tests/test_audio.py +++ b/datalad/metadata/extractors/tests/test_audio.py @@ -11,10 +11,10 @@ from datalad.tests.utils import ( assert_in, assert_not_in, + assert_repo_status, assert_result_count, assert_status, eq_, - ok_clean_git, SkipTest, with_tempfile, ) @@ -53,7 +53,7 @@ def test_audio(path): opj(dirname(dirname(dirname(__file__))), 'tests', 'data', 'audio.mp3'), path) ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) res = ds.aggregate_metadata() assert_status('ok', res) res = ds.metadata('audio.mp3') diff --git a/datalad/metadata/extractors/tests/test_base.py b/datalad/metadata/extractors/tests/test_base.py index 276636620d..1e6b80cfe8 100644 --- a/datalad/metadata/extractors/tests/test_base.py +++ b/datalad/metadata/extractors/tests/test_base.py @@ -13,8 +13,8 @@ from datalad.api import Dataset from datalad.tests.utils import ( assert_equal, + assert_repo_status, known_failure_githubci_win, - ok_clean_git, SkipTest, with_tree, ) @@ -24,7 +24,7 @@ def check_api(no_annex, path): ds = Dataset(path).create(force=True, no_annex=no_annex) ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) processed_extractors, skipped_extractors = [], [] for extractor_ep in iter_entry_points('datalad.metadata.extractors'): diff --git a/datalad/metadata/extractors/tests/test_exif.py b/datalad/metadata/extractors/tests/test_exif.py index 7f45755098..c174044983 100644 --- a/datalad/metadata/extractors/tests/test_exif.py +++ b/datalad/metadata/extractors/tests/test_exif.py @@ -10,10 +10,10 @@ from datalad.tests.utils import ( assert_in, + assert_repo_status, assert_result_count, assert_status, eq_, - ok_clean_git, SkipTest, with_tempfile, ) @@ -86,7 +86,7 @@ def test_exif(path): opj(dirname(dirname(dirname(__file__))), 'tests', 'data', 'exif.jpg'), path) ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) res = ds.aggregate_metadata() assert_status('ok', res) res = ds.metadata('exif.jpg') diff --git a/datalad/metadata/extractors/tests/test_image.py b/datalad/metadata/extractors/tests/test_image.py index 5663b7833f..d7f9f15664 100644 --- a/datalad/metadata/extractors/tests/test_image.py +++ b/datalad/metadata/extractors/tests/test_image.py @@ -10,10 +10,10 @@ from datalad.tests.utils import ( assert_in, + assert_repo_status, assert_result_count, assert_status, eq_, - ok_clean_git, SkipTest, with_tempfile, ) @@ -49,7 +49,7 @@ def test_image(path): opj(dirname(dirname(dirname(__file__))), 'tests', 'data', 'exif.jpg'), path) ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) res = ds.aggregate_metadata() assert_status('ok', res) res = ds.metadata('exif.jpg') diff --git a/datalad/metadata/extractors/tests/test_xmp.py b/datalad/metadata/extractors/tests/test_xmp.py index a23dcb9861..b473b944ea 100644 --- a/datalad/metadata/extractors/tests/test_xmp.py +++ b/datalad/metadata/extractors/tests/test_xmp.py @@ -10,10 +10,10 @@ from datalad.tests.utils import ( assert_in, + assert_repo_status, assert_result_count, assert_status, eq_, - ok_clean_git, SkipTest, with_tempfile, ) @@ -54,7 +54,7 @@ def test_xmp(path): opj(dirname(dirname(dirname(__file__))), 'tests', 'data', 'xmp.pdf'), path) ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) res = ds.aggregate_metadata() assert_status('ok', res) res = ds.metadata('xmp.pdf') diff --git a/datalad/metadata/tests/test_aggregation.py b/datalad/metadata/tests/test_aggregation.py index ec628464a3..9a229c4a6e 100644 --- a/datalad/metadata/tests/test_aggregation.py +++ b/datalad/metadata/tests/test_aggregation.py @@ -19,11 +19,11 @@ from datalad.tests.utils import ( assert_dict_equal, assert_not_in, + assert_repo_status, assert_result_count, assert_status, eq_, known_failure_githubci_win, - ok_clean_git, skip_if_on_windows, skip_ssh, with_tempfile, @@ -67,14 +67,14 @@ def test_basic_aggregate(path): #base.metadata(sub.path, init=dict(homepage='this'), apply2global=True) subsub = base.create(opj('sub', 'subsub'), force=True) base.save(recursive=True) - ok_clean_git(base.path) + assert_repo_status(base.path) # we will first aggregate the middle dataset on its own, this will # serve as a smoke test for the reuse of metadata objects later on sub.aggregate_metadata() base.save() - ok_clean_git(base.path) + assert_repo_status(base.path) base.aggregate_metadata(recursive=True, update_mode='all') - ok_clean_git(base.path) + assert_repo_status(base.path) direct_meta = base.metadata(recursive=True, return_type='list') # loose the deepest dataset sub.uninstall('subsub', check=False) @@ -89,7 +89,7 @@ def test_basic_aggregate(path): # no we can throw away the subdataset tree, and loose no metadata base.uninstall('sub', recursive=True, check=False) assert(not sub.is_installed()) - ok_clean_git(base.path) + assert_repo_status(base.path) # same result for aggregate query than for (saved) direct query agg_meta = base.metadata(recursive=True, return_type='list') for d, a in zip(direct_meta, agg_meta): @@ -160,9 +160,9 @@ def test_reaggregate_with_unavailable_objects(path): sub = base.create('sub', force=True) subsub = base.create(opj('sub', 'subsub'), force=True) base.save(recursive=True) - ok_clean_git(base.path) + assert_repo_status(base.path) base.aggregate_metadata(recursive=True, update_mode='all') - ok_clean_git(base.path) + assert_repo_status(base.path) objpath = opj('.datalad', 'metadata', 'objects') objs = list(sorted(base.repo.find(objpath))) # we have 3x2 metadata sets (dataset/files) under annex @@ -171,7 +171,7 @@ def test_reaggregate_with_unavailable_objects(path): # drop all object content base.drop(objs, check=False) eq_(all(base.repo.file_has_content(objs)), False) - ok_clean_git(base.path) + assert_repo_status(base.path) # now re-aggregate, the state hasn't changed, so the file names will # be the same base.aggregate_metadata(recursive=True, update_mode='all', force_extraction=True) @@ -195,17 +195,17 @@ def test_aggregate_with_unavailable_objects_from_subds(path, target): sub = base.create('sub', force=True) subsub = base.create(opj('sub', 'subsub'), force=True) base.save(recursive=True) - ok_clean_git(base.path) + assert_repo_status(base.path) base.aggregate_metadata(recursive=True, update_mode='all') - ok_clean_git(base.path) + assert_repo_status(base.path) # now make that a subdataset of a new one, so aggregation needs to get the # metadata objects first: super = Dataset(target).create() super.install("base", source=base.path) - ok_clean_git(super.path) + assert_repo_status(super.path) clone = Dataset(opj(super.path, "base")) - ok_clean_git(clone.path) + assert_repo_status(clone.path) objpath = opj('.datalad', 'metadata', 'objects') objs = [o for o in sorted(clone.repo.get_annexed_files(with_content_only=False)) if o.startswith(objpath)] eq_(len(objs), 6) @@ -229,9 +229,9 @@ def test_publish_aggregated(path): '** annex.largefiles=nothing\nmetadata/objects/** annex.largefiles=anything\n') base.create('sub', force=True) base.save(recursive=True) - ok_clean_git(base.path) + assert_repo_status(base.path) base.aggregate_metadata(recursive=True, update_mode='all') - ok_clean_git(base.path) + assert_repo_status(base.path) # create sibling and publish to it spath = opj(path, 'remote') @@ -278,7 +278,7 @@ def test_aggregate_removal(path): subsub = sub.create(opj('subsub'), force=True) base.save(recursive=True) base.aggregate_metadata(recursive=True, update_mode='all') - ok_clean_git(base.path) + assert_repo_status(base.path) res = base.metadata(get_aggregates=True) assert_result_count(res, 3) assert_result_count(res, 1, path=subsub.path) @@ -290,7 +290,7 @@ def test_aggregate_removal(path): # now aggregation has to detect that subsub is not simply missing, but gone # for good base.aggregate_metadata(recursive=True, update_mode='all') - ok_clean_git(base.path) + assert_repo_status(base.path) # internally consistent state eq_(_get_contained_objs(base), _get_referenced_objs(base)) # info on subsub was removed at all levels @@ -315,7 +315,7 @@ def test_update_strategy(path): sub = base.create('sub', force=True) subsub = sub.create(opj('subsub'), force=True) base.save(recursive=True) - ok_clean_git(base.path) + assert_repo_status(base.path) # we start clean for ds in base, sub, subsub: eq_(len(_get_contained_objs(ds)), 0) @@ -404,7 +404,7 @@ def test_partial_aggregation(path): with open(opj(sub1.path, 'here'), 'w') as f: f.write('fresh') ds.save(recursive=True) - ok_clean_git(path) + assert_repo_status(path) # TODO for later # test --since with non-incremental #ds.aggregate_metadata(recursive=True, since='HEAD~1', incremental=False) diff --git a/datalad/metadata/tests/test_base.py b/datalad/metadata/tests/test_base.py index 76aff63171..826ea665d9 100644 --- a/datalad/metadata/tests/test_base.py +++ b/datalad/metadata/tests/test_base.py @@ -38,12 +38,12 @@ assert_in, assert_raises, assert_re_in, + assert_repo_status, assert_result_count, assert_status, assert_true, eq_, known_failure_githubci_win, - ok_clean_git, slow, swallow_logs, with_tempfile, @@ -127,7 +127,7 @@ def test_aggregation(path): subsubds.config.add('datalad.metadata.nativetype', 'frictionless_datapackage', where='dataset') ds.save(recursive=True) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # aggregate metadata from all subdatasets into any superdataset, including # intermediate ones res = ds.aggregate_metadata(recursive=True, update_mode='all') @@ -137,7 +137,7 @@ def test_aggregation(path): assert_result_count(res, 3, status='ok', action='aggregate_metadata') assert_result_count(res, 3, status='ok', action='save') # nice and tidy - ok_clean_git(ds.path) + assert_repo_status(ds.path) # quick test of aggregate report aggs = ds.metadata(get_aggregates=True) diff --git a/datalad/metadata/tests/test_extract_metadata.py b/datalad/metadata/tests/test_extract_metadata.py index ed5673e594..3127ef4a9a 100644 --- a/datalad/metadata/tests/test_extract_metadata.py +++ b/datalad/metadata/tests/test_extract_metadata.py @@ -23,9 +23,9 @@ from datalad.tests.utils import ( assert_in, assert_raises, + assert_repo_status, assert_result_count, known_failure_githubci_win, - ok_clean_git, with_tempfile, ) @@ -52,7 +52,7 @@ def test_ds_extraction(path): ds = Dataset(path).create() copy(testpath, path) ds.save() - ok_clean_git(ds.path) + assert_repo_status(ds.path) res = extract_metadata( types=['xmp'], diff --git a/datalad/metadata/tests/test_search.py b/datalad/metadata/tests/test_search.py index ef99ff1727..b55d1189f7 100644 --- a/datalad/metadata/tests/test_search.py +++ b/datalad/metadata/tests/test_search.py @@ -28,10 +28,10 @@ assert_in, assert_is_generator, assert_raises, + assert_repo_status, assert_result_count, eq_, known_failure_githubci_win, - ok_clean_git, ok_file_under_git, patch_config, SkipTest, @@ -213,7 +213,7 @@ def test_within_ds_file_search(path): ds.repo.set_metadata( opj('stim', 'stim1.mp3'), init={'importance': 'very'}) ds.aggregate_metadata() - ok_clean_git(ds.path) + assert_repo_status(ds.path) # basic sanity check on the metadata structure of the dataset dsmeta = ds.metadata('.', reporton='datasets')[0]['metadata'] for src in ('audio',): diff --git a/datalad/plugin/tests/test_plugins.py b/datalad/plugin/tests/test_plugins.py index 1c66de9a40..2d599be06a 100644 --- a/datalad/plugin/tests/test_plugins.py +++ b/datalad/plugin/tests/test_plugins.py @@ -29,13 +29,13 @@ from datalad.tests.utils import ( assert_in, assert_not_in, + assert_repo_status, assert_status, chpwd, create_tree, eq_, known_failure_githubci_win, OBSCURE_FILENAME, - ok_clean_git, ok_startswith, skip_if_no_module, SkipTest, @@ -168,7 +168,7 @@ def test_wtf(topdir): @with_tempfile(mkdir=True) def test_no_annex(path): ds = create(path) - ok_clean_git(ds.path) + assert_repo_status(ds.path) create_tree( ds.path, {'code': { @@ -180,7 +180,7 @@ def test_no_annex(path): no_annex(pattern=['code/**', 'README'], dataset=ds.path) # add inannex and README post configuration ds.save([opj('code', 'notinannex'), 'README']) - ok_clean_git(ds.path) + assert_repo_status(ds.path) # one is annex'ed, the other is not, despite no change in add call # importantly, also .gitattribute is not annexed eq_([opj('code', 'inannex')], @@ -212,7 +212,7 @@ def test_add_readme(path): ds = Dataset(path).create(force=True) ds.save() ds.aggregate_metadata() - ok_clean_git(ds.path) + assert_repo_status(ds.path) assert_status('ok', ds.add_readme()) # should use default name eq_( diff --git a/datalad/support/tests/test_annexrepo.py b/datalad/support/tests/test_annexrepo.py index 274ac1eeeb..86af9b647c 100644 --- a/datalad/support/tests/test_annexrepo.py +++ b/datalad/support/tests/test_annexrepo.py @@ -80,7 +80,6 @@ OBSCURE_FILENAME, ok_, ok_annex_get, - ok_clean_git, ok_file_has_content, ok_file_under_git, ok_git_config_not_empty, @@ -778,10 +777,10 @@ def test_AnnexRepo_commit(path): f.write("File to add to git") ds.add(filename, git=True) - assert_raises(AssertionError, ok_clean_git, path, annex=True) + assert_raises(AssertionError, assert_repo_status, path, annex=True) ds.commit("test _commit") - ok_clean_git(path, annex=True) + assert_repo_status(path, annex=True) # nothing to commit doesn't raise by default: ds.commit() @@ -807,7 +806,7 @@ def test_AnnexRepo_add_to_annex(path): # clone as provided by with_testrepos: repo = AnnexRepo(path, create=False, init=True) - ok_clean_git(repo, annex=True, ignore_submodules=True) + assert_repo_status(repo, annex=True) filename = get_most_obscure_supported_name() filename_abs = opj(repo.path, filename) with open(filename_abs, "w") as f: @@ -827,7 +826,7 @@ def test_AnnexRepo_add_to_annex(path): ok_(repo.dirty) repo.commit("Added file to annex.") - ok_clean_git(repo, annex=True, ignore_submodules=True) + assert_repo_status(repo, annex=True) # now using commit/msg options: filename = "another.txt" @@ -841,7 +840,7 @@ def test_AnnexRepo_add_to_annex(path): ok_(repo.file_has_content(filename)) # and committed: - ok_clean_git(repo, annex=True, ignore_submodules=True) + assert_repo_status(repo, annex=True) @with_testrepos('.*annex.*', flavors=['clone']) @@ -854,7 +853,7 @@ def test_AnnexRepo_add_to_git(path): # clone as provided by with_testrepos: repo = AnnexRepo(path, create=False, init=True) - ok_clean_git(repo, annex=True, ignore_submodules=True) + assert_repo_status(repo, annex=True) filename = get_most_obscure_supported_name() with open(opj(repo.path, filename), "w") as f: f.write("some") @@ -865,7 +864,7 @@ def test_AnnexRepo_add_to_git(path): # uncommitted: ok_(repo.dirty) repo.commit("Added file to annex.") - ok_clean_git(repo, annex=True, ignore_submodules=True) + assert_repo_status(repo, annex=True) # now using commit/msg options: filename = "another.txt" @@ -878,7 +877,7 @@ def test_AnnexRepo_add_to_git(path): assert_raises(FileInGitError, repo.get_file_key, filename) # and committed: - ok_clean_git(repo, annex=True, ignore_submodules=True) + assert_repo_status(repo, annex=True) @with_testrepos('.*annex.*', flavors=['local']) @@ -1256,7 +1255,7 @@ def test_annex_remove(path1, path2): @with_tempfile def test_repo_version(path1, path2, path3): annex = AnnexRepo(path1, create=True, version=6) - ok_clean_git(path1, annex=True) + assert_repo_status(path1, annex=True) version = int(annex.config.get('annex.version')) # Since git-annex 7.20181031, v6 repos upgrade to v7. supported_versions = AnnexRepo.check_repository_versions()["supported"] @@ -1749,8 +1748,8 @@ def test_AnnexRepo_add_submodule(source, path): top_repo.commit('submodule added') eq_([s.name for s in top_repo.get_submodules()], ['sub']) - ok_clean_git(top_repo, annex=True) - ok_clean_git(opj(path, 'sub'), annex=False) + assert_repo_status(top_repo, annex=True) + assert_repo_status(opj(path, 'sub'), annex=False) def test_AnnexRepo_update_submodule(): @@ -1822,9 +1821,6 @@ def test_AnnexRepo_dirty(path): ok_(not repo.dirty) -# TODO: test/utils ok_clean_git - - @with_tempfile(mkdir=True) def test_AnnexRepo_set_remote_url(path): @@ -1890,7 +1886,7 @@ def test_AnnexRepo_metadata(path): }) ar.add('.', git=False) ar.commit('content') - ok_clean_git(path) + assert_repo_status(path) # fugue # doesn't do anything if there is nothing to do ar.set_metadata('up.dat') @@ -2166,7 +2162,7 @@ def check_commit_annex_commit_changed(unlock, path): ar = AnnexRepo(path, create=True) ar.save("initial commit") - ok_clean_git(path) + assert_repo_status(path) # Now let's change all but commit only some files = [op.basename(p) for p in glob(op.join(path, '*'))] if unlock: @@ -2184,18 +2180,18 @@ def check_commit_annex_commit_changed(unlock, path): } , remove_existing=True ) - ok_clean_git( + assert_repo_status( path - , index_modified=files if not unannex else ['tobechanged-git'] + , modified=files if not unannex else ['tobechanged-git'] , untracked=['untracked'] if not unannex else # all but the one in git now ['alwaysbig', 'tobechanged-annex', 'untracked', 'willgetshort'] ) ar.save("message", paths=['alwaysbig', 'willgetshort']) - ok_clean_git( + assert_repo_status( path - , index_modified=['tobechanged-git', 'tobechanged-annex'] + , modified=['tobechanged-git', 'tobechanged-annex'] , untracked=['untracked'] ) ok_file_under_git(path, 'alwaysbig', annexed=True) @@ -2205,7 +2201,7 @@ def check_commit_annex_commit_changed(unlock, path): annexed=external_versions['cmd:annex']<'7.20191009') ar.save("message2", untracked='no') # commit all changed - ok_clean_git( + assert_repo_status( path , untracked=['untracked'] ) diff --git a/datalad/support/tests/test_gitrepo.py b/datalad/support/tests/test_gitrepo.py index f3b0d03d5a..54c2255243 100644 --- a/datalad/support/tests/test_gitrepo.py +++ b/datalad/support/tests/test_gitrepo.py @@ -36,6 +36,7 @@ assert_in_results, assert_not_in, assert_raises, + assert_repo_status, create_tree, eq_, get_most_obscure_supported_name, @@ -44,7 +45,6 @@ local_testrepo_flavors, neq_, ok_, - ok_clean_git, skip_if, skip_if_no_network, skip_if_on_windows, @@ -129,13 +129,13 @@ def test_GitRepo_instance_from_not_existing(path, path2): gr = GitRepo(path2, create=True) assert_is_instance(gr, GitRepo, "GitRepo was not created.") ok_(op.exists(op.join(path2, '.git'))) - ok_clean_git(path2, annex=False) + assert_repo_status(path2, annex=False) # 4. create=True, path exists, but no git repo: gr = GitRepo(path, create=True) assert_is_instance(gr, GitRepo, "GitRepo was not created.") ok_(op.exists(op.join(path, '.git'))) - ok_clean_git(path, annex=False) + assert_repo_status(path, annex=False) @with_tempfile @@ -212,7 +212,7 @@ def test_GitRepo_add(src, path): assert_in(filename, gr.get_indexed_files(), "%s not successfully added to %s" % (filename, path)) - ok_clean_git(path) + assert_repo_status(path) @assert_cwd_unchanged @@ -248,7 +248,7 @@ def test_GitRepo_commit(path): gr.add(filename) gr.commit("Testing GitRepo.commit().") - ok_clean_git(gr) + assert_repo_status(gr) eq_("Testing GitRepo.commit().", gr.format_commit("%B").strip()) @@ -262,7 +262,7 @@ def test_GitRepo_commit(path): # commit with empty message: gr.commit() - ok_clean_git(gr) + assert_repo_status(gr) # nothing to commit doesn't raise by default: gr.commit() @@ -504,7 +504,7 @@ def test_GitRepo_fetch(test_path, orig_path, clone_path): eq_([u'origin/' + clone.get_active_branch(), u'origin/new_branch'], [commit['ref'] for commit in fetched]) - ok_clean_git(clone.path, annex=False) + assert_repo_status(clone.path, annex=False) assert_in("origin/new_branch", clone.get_remote_branches()) assert_in(filename, clone.get_files("origin/new_branch")) assert_false(op.exists(op.join(clone_path, filename))) # not checked out @@ -549,7 +549,7 @@ def test_GitRepo_ssh_fetch(remote_path, repo_path): fetched = repo.fetch(remote="ssh-remote") assert_in('ssh-remote/master', [commit['ref'] for commit in fetched]) - ok_clean_git(repo) + assert_repo_status(repo) # the connection is known to the SSH manager, since fetch() requested it: assert_in(socket_path, list(map(str, ssh_manager._connections))) @@ -588,7 +588,7 @@ def test_GitRepo_ssh_pull(remote_path, repo_path): # pull changes: repo.pull(remote="ssh-remote", refspec=remote_repo.get_active_branch()) - ok_clean_git(repo.path, annex=False) + assert_repo_status(repo.path, annex=False) # the connection is known to the SSH manager, since fetch() requested it: assert_in(socket_path, list(map(str, ssh_manager._connections))) @@ -971,8 +971,8 @@ def test_GitRepo_add_submodule(source, path): top_repo.add_submodule('sub', name='sub', url=source) top_repo.commit('submodule added') eq_([s.name for s in top_repo.get_submodules()], ['sub']) - ok_clean_git(path) - ok_clean_git(op.join(path, 'sub')) + assert_repo_status(path) + assert_repo_status(op.join(path, 'sub')) def test_GitRepo_update_submodule(): @@ -1116,7 +1116,7 @@ def test_optimized_cloning(path): f.write('some') repo.add('test') repo.commit('init') - ok_clean_git(originpath, annex=False) + assert_repo_status(originpath, annex=False) from glob import glob def _get_inodes(repo): diff --git a/datalad/tests/test_utils_testrepos.py b/datalad/tests/test_utils_testrepos.py index d8ff20d86f..ea76dbfdad 100644 --- a/datalad/tests/test_utils_testrepos.py +++ b/datalad/tests/test_utils_testrepos.py @@ -14,8 +14,8 @@ BasicGitTestRepo, ) from datalad.tests.utils import ( + assert_repo_status, ok_, - ok_clean_git, ok_file_under_git, skip_if_on_windows, swallow_outputs, @@ -26,7 +26,7 @@ def _test_BasicAnnexTestRepo(repodir): trepo = BasicAnnexTestRepo(repodir) trepo.create() - ok_clean_git(trepo.path) + assert_repo_status(trepo.path) ok_file_under_git(trepo.path, 'test.dat') ok_file_under_git(trepo.path, 'INFO.txt') ok_file_under_git(trepo.path, 'test-annex.dat', annexed=True) @@ -55,6 +55,6 @@ def test_BasicAnnexTestRepo(path): def test_BasicGitTestRepo(path): trepo = BasicGitTestRepo(path) trepo.create() - ok_clean_git(trepo.path, annex=False) + assert_repo_status(trepo.path, annex=False) ok_file_under_git(trepo.path, 'test.dat') ok_file_under_git(trepo.path, 'INFO.txt') diff --git a/datalad/tests/utils.py b/datalad/tests/utils.py index b21d4f6b23..96d87cd333 100644 --- a/datalad/tests/utils.py +++ b/datalad/tests/utils.py @@ -270,7 +270,6 @@ def newfunc(*args, **kwargs): # Addition "checkers" # -import git import os from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo, FileNotInAnnexError @@ -278,90 +277,22 @@ def newfunc(*args, **kwargs): from ..utils import chpwd, getpwd -def ok_clean_git(path, annex=None, head_modified=[], index_modified=[], - untracked=[], ignore_submodules=False): - """Verify that under given path there is a clean git repository +def ok_clean_git(path, annex=None, index_modified=[], untracked=[]): + """Obsolete test helper. Use assert_repo_status() instead. - it exists, .git exists, nothing is uncommitted/dirty/staged - - Note - ---- - Parameters head_modified and index_modified currently work - in pure git or indirect mode annex only. If they are given, no - test of modification of known repo content is performed. - - Parameters - ---------- - path: str or Repo - in case of a str: path to the repository's base dir; - Note, that passing a Repo instance prevents detecting annex. This might be - useful in case of a non-initialized annex, a GitRepo is pointing to. - annex: bool or None - explicitly set to True or False to indicate, that an annex is (not) - expected; set to None to autodetect, whether there is an annex. - Default: None. - ignore_submodules: bool - if True, submodules are not inspected + Still maps a few common cases to the new helper, to ease transition + in extensions. """ - # TODO: See 'Note' in docstring - - if isinstance(path, AnnexRepo): - if annex is None: - annex = True - # if `annex` was set to False, but we find an annex => fail - assert_is(annex, True) - r = path - elif isinstance(path, GitRepo): - if annex is None: - annex = False - # explicitly given GitRepo instance doesn't make sense with 'annex' True - assert_is(annex, False) - r = path - else: - # 'path' is an actual path - try: - r = AnnexRepo(path, init=False, create=False) - if annex is None: - annex = True - # if `annex` was set to False, but we find an annex => fail - assert_is(annex, True) - except Exception: - # Instantiation failed => no annex - try: - r = GitRepo(path, init=False, create=False) - except Exception: - raise AssertionError("Couldn't find an annex or a git " - "repository at {}.".format(path)) - if annex is None: - annex = False - # explicitly given GitRepo instance doesn't make sense with - # 'annex' True - assert_is(annex, False) - - eq_(sorted(r.untracked_files), sorted(untracked)) - - import git - repo = git.Repo(r.path) - - if repo.index.entries.keys(): - ok_(repo.head.is_valid()) - - if not head_modified and not index_modified: - # get string representations of diffs with index to ease - # troubleshooting - head_diffs = [str(d) for d in repo.index.diff(repo.head.commit)] - index_diffs = [str(d) for d in repo.index.diff(None)] - eq_(head_diffs, []) - eq_(index_diffs, []) - else: - # TODO: These names are confusing/non-descriptive. REDO - if head_modified: - # we did ask for interrogating changes - head_modified_ = [d.a_path for d in repo.index.diff(repo.head.commit)] - eq_(sorted(head_modified_), sorted(head_modified)) - if index_modified: - index_modified_ = [d.a_path for d in repo.index.diff(None)] - eq_(sorted(index_modified_), sorted(index_modified)) + kwargs = {} + if index_modified: + kwargs['modified'] = index_modified + if untracked: + kwargs['untracked'] = untracked + assert_repo_status( + path, + annex=annex, + **kwargs, + ) def ok_file_under_git(path, filename=None, annexed=False): @@ -1578,8 +1509,6 @@ def assert_repo_status(path, annex=None, untracked_mode='normal', **kwargs): Anything file/directory that is not explicitly indicated must have state 'clean', i.e. no modifications and recorded in Git. - This is an alternative to the traditional `ok_clean_git` helper. - Parameters ---------- path: str or Repo From 942de71458dec6d0b39d148b88932f43f634de0f Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Wed, 26 Feb 2020 07:53:03 +0100 Subject: [PATCH 07/10] RF: Remove last (unused) import and strip GitPython from dependencies --- datalad/support/tests/test_annexrepo.py | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/datalad/support/tests/test_annexrepo.py b/datalad/support/tests/test_annexrepo.py index 86af9b647c..5c9842e730 100644 --- a/datalad/support/tests/test_annexrepo.py +++ b/datalad/support/tests/test_annexrepo.py @@ -34,7 +34,6 @@ from urllib.parse import urljoin from urllib.parse import urlsplit -import git from unittest.mock import patch import gc diff --git a/setup.py b/setup.py index 2eca13f82c..7af9bbc3f4 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ 'chardet>=3.0.4', # rarely used but small/omnipresent 'colorama; platform_system=="Windows"', 'distro; python_version >= "3.8"', - 'GitPython>=2.1.12', 'iso8601', 'humanize', 'fasteners', From c91ba50ca5fd2964a42ec6547fc0a096f8c8bd70 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Wed, 26 Feb 2020 08:24:01 +0100 Subject: [PATCH 08/10] TST: Strip now obsolete travis run for upstream GitPython --- .travis.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e717fb000..870896a0c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -106,16 +106,6 @@ matrix: # The ones to run only on weekends against master. # They will not contribute to coverage etc, but might lead to failed status # - - python: 3.5 - # By default no logs will be output. This one is to test with log output at INFO level - env: - - _DL_UPSTREAM_GITPYTHON=1 - # It is not usable without setting locales env vars ATM - # see https://github.com/datalad/datalad/issues/3159 - # - _DL_UPSTREAM_GITANNEX=1 - # Just so we test if we did not screw up running under nose without -s as well - - NOSE_OPTS= - - _DL_CRON=1 # run if git-annex version in neurodebian -devel differs - python: 3.5 env: @@ -206,7 +196,6 @@ before_install: # Install grunt-cli - eatmydata npm install grunt-cli # Install optionally upstream current development so we are sure that they break nothing important for us - - if [ ! -z "${_DL_UPSTREAM_GITPYTHON:-}" ]; then pip install https://github.com/gitpython-developers/GitPython/archive/master.zip; fi - if [ ! -z "${_DL_UPSTREAM_GITANNEX:-}" ]; then sudo tools/ci/install-annex-snapshot.sh; sudo ln -s `find /usr/local/lib/git-annex.linux -maxdepth 1 -type f -perm /+x` /usr/local/bin/; else sudo eatmydata apt-get install git-annex-standalone ; fi # Install optionally -devel version of annex, and if goes wrong (we have most recent), exit right away - if [ ! -z "${_DL_DEVEL_ANNEX:-}" ]; then tools/ci/prep-travis-devel-annex.sh || { ex="$?"; if [ "$ex" -eq 99 ]; then exit 0; else exit "$ex"; fi; }; fi From 39c6c30791f8da219837e488f7a7f0d690332287 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Wed, 4 Mar 2020 07:10:09 +0100 Subject: [PATCH 09/10] TST: Disable SSH-based tests on NFS mount Prevents a stall of `annex sync`(?) in datalad.support.tests.test_annexrepo.test_annex_ssh on Travis that does not replicate locally. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 870896a0c3..7dc8dc330a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,6 +100,9 @@ matrix: - python: 3.5 # Test some under NFS mount (only selected sub-set) env: + # do not run SSH-based tests due to stall(s) + # https://github.com/datalad/datalad/pull/4172 + - DATALAD_TESTS_SSH=0 - TMPDIR="/tmp/nfsmount" - TESTS_TO_PERFORM="datalad.tests datalad.support" # From 01bc3018c5fe47d9a2fb50b6c617de2afb8f6445 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Fri, 6 Mar 2020 15:48:13 +0100 Subject: [PATCH 10/10] RF: Remove obsolete test for GitPython version --- datalad/distribution/tests/test_uninstall.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/datalad/distribution/tests/test_uninstall.py b/datalad/distribution/tests/test_uninstall.py index 08d2727a30..cbaa18dbab 100644 --- a/datalad/distribution/tests/test_uninstall.py +++ b/datalad/distribution/tests/test_uninstall.py @@ -219,11 +219,6 @@ def test_uninstall_subdataset(src, dst): for subds in ds.subdatasets(result_xfm='datasets'): # uninstall subds itself: - if os.environ.get('DATALAD_TESTS_DATALADREMOTE') \ - and external_versions['git'] < '2.0.9': - raise SkipTest( - "Known problem with GitPython. See " - "https://github.com/gitpython-developers/GitPython/pull/521") # simulate a cmdline invocation pointing to the subdataset # with a relative path from outside the superdataset to catch # https://github.com/datalad/datalad/issues/4001