From d64d68f65d06b3d453ed4ecf3b8d2d831d86eb9a Mon Sep 17 00:00:00 2001 From: vtemian Date: Mon, 8 Dec 2014 16:31:35 +0200 Subject: [PATCH 01/11] fix #165: append new entry in gitignore, dont add them --- gitfs/cache/gitignore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index 9ca7155d..0d8c8759 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -43,7 +43,7 @@ def update(self): for item in gitignore.readlines(): item = item.strip() if item and not item.startswith('#'): - self.items += item + self.items.append(item) if self.submodules and os.path.exists(self.submodules): with open(self.submodules) as submodules: From 71c4894d239341ec47b8cb08f458f736a157e648 Mon Sep 17 00:00:00 2001 From: vtemian Date: Mon, 8 Dec 2014 17:37:11 +0200 Subject: [PATCH 02/11] fix #168: implement link method --- gitfs/views/current.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/gitfs/views/current.py b/gitfs/views/current.py index f05d82aa..5d2d74a6 100644 --- a/gitfs/views/current.py +++ b/gitfs/views/current.py @@ -61,6 +61,20 @@ def symlink(self, name, target): log.debug("CurrentView: Created symlink to %s from %s", name, target) return result + @write_operation + @not_in("ignore", check=["target"]) + def link(self, name, target): + if target.startswith('/current/'): + target = target.replace('/current/', '/') + + result = super(CurrentView, self).link(target, name) + + message = "Create link to %s for %s" % (target, name) + self._stage(add=name, message=message) + + log.debug("CurrentView: Created link to %s from %s", name, target) + return result + def readlink(self, path): log.debug("CurrentView: Read link %s", path) return os.readlink(self.repo._full_path(path)) From 92d10458925b3d7fc3632a44c9401aab469c2374 Mon Sep 17 00:00:00 2001 From: vtemian Date: Mon, 8 Dec 2014 19:08:33 +0200 Subject: [PATCH 03/11] fix #166: improve _stage method --- gitfs/views/current.py | 45 +++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/gitfs/views/current.py b/gitfs/views/current.py index 5d2d74a6..c3ea0e13 100644 --- a/gitfs/views/current.py +++ b/gitfs/views/current.py @@ -255,21 +255,52 @@ def unlink(self, path): def _stage(self, message, add=None, remove=None): non_empty = False - add = self._sanitize(add) - remove = self._sanitize(remove) - if remove is not None: - self.repo.index.remove(self._sanitize(remove)) + remove = self._sanitize(remove) + if add is not None: + add = self._sanitize(add) + paths = self._get_files_from_path(add) + if paths: + for path in paths: + path = path.replace("%s/" % add, "%s/" % remove) + self.repo.index.remove(path) + else: + self.repo.index.remove(remove) + else: + self.repo.index.remove(remove) non_empty = True if add is not None: - self.repo.index.add(self._sanitize(add)) + add = self._sanitize(add) + paths = self._get_files_from_path(add) + if paths: + for path in paths: + self.repo.index.add(path) + else: + self.repo.index.add(add) non_empty = True if non_empty: self.queue.commit(add=add, remove=remove, message=message) + def _get_files_from_path(self, path): + paths = [] + + full_path = self.repo._full_path(self._sanitize(path)) + workdir = self.repo._repo.workdir + + if os.path.isdir(full_path): + for (dirpath, dirs, files) in os.walk(full_path): + for filename in files: + paths.append("%s/%s" % (dirpath.replace(workdir, ''), + filename)) + return paths + def _sanitize(self, path): - if path is not None and path.startswith("/"): - path = path[1:] + if path is None: + return path + + if path.startswith("/"): + return path[1:] + return path From a5e40277d643d5784fe13d3e92355fbae9e2d2ea Mon Sep 17 00:00:00 2001 From: vtemian Date: Tue, 9 Dec 2014 11:33:04 +0200 Subject: [PATCH 04/11] fix #167: added .gitignore and .gitmodules in IgnoreCache --- gitfs/cache/gitignore.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index 0d8c8759..d879c63e 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -36,7 +36,8 @@ def __init__(self, ignore=False, submodules=False, path="."): self.update() def update(self): - self.items = ['.git', '.git/*', '/.git/*', '*.keep'] + self.items = ['.git', '.git/*', '/.git/*', '*.keep', + '*.gitignore', '*.gitmodules'] if self.ignore and os.path.exists(self.ignore): with open(self.ignore) as gitignore: From 853e369ee012229f1634b844c5a7a86cae08e7fe Mon Sep 17 00:00:00 2001 From: vtemian Date: Tue, 9 Dec 2014 12:06:29 +0200 Subject: [PATCH 05/11] Close #164: Added ignore_file, module_file and hard_ignore as options --- gitfs/cache/gitignore.py | 19 +++++++++++-------- gitfs/mounter.py | 5 ++++- gitfs/router.py | 5 +++-- gitfs/utils/args.py | 7 +++++++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index d879c63e..b017d9bd 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -19,19 +19,15 @@ class CachedIgnore(object): - def __init__(self, ignore=False, submodules=False, path="."): + def __init__(self, ignore=False, submodules=False, hard_ignore=None): self.items = [] - self.ignore = False - if ignore: - self.ignore = os.path.join(path, ".gitignore") - - self.submodules = False - if submodules: - self.submodules = os.path.join(path, ".gitmodules") + self.ignore = ignore + self.submodules = submodules self.cache = {} self.permanent = [] + self.hard_ignore = self._parse_hard_ignore(hard_ignore) self.update() @@ -57,6 +53,13 @@ def update(self): self.items.append("%s" % result[2]) self.cache = {} + self.items += self.hard_ignore + + def _parse_hard_ignore(self, hard_ignore): + if isinstance(hard_ignore, basestring): + return hard_ignore.split("|") + else: + return [] def __contains__(self, path): return self.check_key(path) diff --git a/gitfs/mounter.py b/gitfs/mounter.py index edea6e8b..a98b4ee5 100644 --- a/gitfs/mounter.py +++ b/gitfs/mounter.py @@ -61,7 +61,10 @@ def prepare_components(args): max_size=args.max_size * 1024 * 1024, max_offset=args.max_size * 1024 * 1024, commit_queue=commit_queue, - credentials=credentials) + credentials=credentials, + module_file=args.module_file, + ignore_file=args.ignore_file, + hard_ignore=args.hard_ignore) # register all the routes router.register(routes) diff --git a/gitfs/router.py b/gitfs/router.py index 3e0ad9c8..6c8ca0de 100644 --- a/gitfs/router.py +++ b/gitfs/router.py @@ -59,8 +59,9 @@ def __init__(self, remote_url, repo_path, mount_path, credentials, log.info('Done cloning') self.repo.credentials = credentials - self.repo.ignore = CachedIgnore(submodules=True, ignore=True, - path=self.repo_path) + self.repo.ignore = CachedIgnore(submodules=kwargs['module_file'], + ignore=kwargs['ignore_file'], + hard_ignore=kwargs['hard_ignore']) self.uid = getpwnam(user).pw_uid self.gid = getgrnam(group).gr_gid diff --git a/gitfs/utils/args.py b/gitfs/utils/args.py index 02a520c3..69d55fa4 100644 --- a/gitfs/utils/args.py +++ b/gitfs/utils/args.py @@ -53,6 +53,10 @@ def __init__(self, parser): ("log_level", ("warning", "string")), ("cache_size", (800, "int")), ("sentry_dsn", (self.get_sentry_dsn, "string")), + ("ignore", ("", "string")), + ("ignore_file", (self.get_dot_file('.gitignore'), "string")), + ("module_file", (self.get_dot_file('.gitmodules'), "string")), + ("hard_ignore", ("", "string")), ]) self.config = self.build_config(parser.parse_args()) @@ -161,6 +165,9 @@ def get_repo_path(self, args): def get_ssh_key(self, args): return os.environ["HOME"] + "/.ssh/id_rsa" + def get_dot_file(self, default_file): + return lambda args: os.path.join(args.repo_path, default_file) + def get_sentry_dsn(self, args): return os.environ["SENTRY_DSN"] if "SENTRY_DSN" in os.environ else "" From a115e676d135233750434ea93049ca84f1ad0cf7 Mon Sep 17 00:00:00 2001 From: vtemian Date: Tue, 9 Dec 2014 17:17:34 +0200 Subject: [PATCH 06/11] fixed tests --- tests/cache/test_gitignore.py | 3 ++- tests/test_mount.py | 7 +++++-- tests/test_router.py | 9 ++++++--- tests/views/test_current.py | 3 +++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/cache/test_gitignore.py b/tests/cache/test_gitignore.py index b68ddecf..3ac2243e 100644 --- a/tests/cache/test_gitignore.py +++ b/tests/cache/test_gitignore.py @@ -33,7 +33,8 @@ def test_init(self): gitignore = CachedIgnore("some_file", "some_file") assert gitignore.items == ['.git', '.git/*', '/.git/*', - '*.keep', '/found/*', + '*.keep', '*.gitignore', + '*.gitmodules', '/found/*', '/found', 'found'] def test_update(self): diff --git a/tests/test_mount.py b/tests/test_mount.py index 5743e065..d12a70a4 100644 --- a/tests/test_mount.py +++ b/tests/test_mount.py @@ -60,7 +60,10 @@ def test_prepare_components(self): 'merge_timeout': 10, 'commiter_name': 'commit', 'commiter_email': 'commiter@commiting.org', - 'log': 'syslog' + 'log': 'syslog', + 'ignore_file': '', + 'module_file': '', + 'hard_ignore': None, }) mocked_argparse.Argumentparser.return_value = mocked_parser @@ -91,7 +94,7 @@ def test_prepare_components(self): 'branch': 'branch', 'timeout': 10, 'repo_path': 'repo_path', - 'commit_queue': mocked_queue + 'commit_queue': mocked_queue, } mocked_merger.assert_called_once_with('commit', 'commiter@commiting.org', diff --git a/tests/test_router.py b/tests/test_router.py index 2f6823ee..04304d6f 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -72,6 +72,9 @@ def get_new_router(self): 'commit_queue': mocked_queue, 'max_size': 10, 'max_offset': 10, + 'ignore_file': '', + 'module_file': '', + 'hard_ignore': None, } with patch.multiple('gitfs.router', Repository=mocked_repository, @@ -92,9 +95,9 @@ def test_constructor(self): mocks['branch'], mocks['credentials']) mocks['repository'].clone.assert_called_once_with(*asserted_call) mocks['ignore'].assert_called_once_with(**{ - 'submodules': True, - 'ignore': True, - 'path': mocks['repo_path'], + 'ignore': '', + 'hard_ignore': None, + 'submodules': '', }) mocks['getpwnam'].assert_called_once_with(mocks['user']) mocks['getgrnam'].assert_called_once_with(mocks['group']) diff --git a/tests/views/test_current.py b/tests/views/test_current.py index fd73b253..993ebb74 100644 --- a/tests/views/test_current.py +++ b/tests/views/test_current.py @@ -360,6 +360,7 @@ def test_stage(self): mocked_repo = MagicMock() mocked_sanitize = MagicMock() mocked_queue = MagicMock() + mocked_files = MagicMock(return_value=None) mocked_sanitize.return_value = ["to-stage"] @@ -367,6 +368,7 @@ def test_stage(self): repo_path="repo_path", queue=mocked_queue, ignore=CachedIgnore()) current._sanitize = mocked_sanitize + current._get_files_from_path = mocked_files current._stage("message", ["add"], ["remove"]) mocked_queue.commit.assert_called_once_with(add=['to-stage'], @@ -375,6 +377,7 @@ def test_stage(self): mocked_repo.index.add.assert_called_once_with(["to-stage"]) mocked_repo.index.remove.assert_called_once_with(["to-stage"]) + mocked_files.has_calls([call(['add'])]) mocked_sanitize.has_calls([call(['add']), call(['remove'])]) def test_sanitize(self): From 57fc9ba9e7db4933d4914fd3a8d2afa350199b19 Mon Sep 17 00:00:00 2001 From: vtemian Date: Tue, 9 Dec 2014 17:40:48 +0200 Subject: [PATCH 07/11] Close #170: added tests for hard link --- tests/integrations/current/test_write.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/integrations/current/test_write.py b/tests/integrations/current/test_write.py index e4e6fc6f..c15a9117 100644 --- a/tests/integrations/current/test_write.py +++ b/tests/integrations/current/test_write.py @@ -19,12 +19,26 @@ import string import shutil -import pytest - from tests.integrations.base import BaseTest, pull class TestWriteCurrentView(BaseTest): + def test_link_a_file(self): + filename = "%s/link_file" % self.current_path + link_name = "%s/new_link" % self.current_path + + with open(filename, "w") as f: + f.write("some content") + + os.link(filename, link_name) + + time.sleep(5) + with pull(self.sh): + self.assert_commit_message("Update 2 items") + + is_link = os.path.isfile(link_name) + assert is_link is not False + def test_write_a_file(self): content = "Just a small file" filename = "%s/new_file" % self.current_path From 65353fc04d8a34dd2ebd672b5e71633eb2f32a22 Mon Sep 17 00:00:00 2001 From: vtemian Date: Tue, 9 Dec 2014 17:51:35 +0200 Subject: [PATCH 08/11] Close #169: added test for removing a direcotry --- tests/integrations/current/test_write.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integrations/current/test_write.py b/tests/integrations/current/test_write.py index c15a9117..bd48e9ea 100644 --- a/tests/integrations/current/test_write.py +++ b/tests/integrations/current/test_write.py @@ -23,6 +23,24 @@ class TestWriteCurrentView(BaseTest): + def test_rename_directory(self): + old_dir = "%s/a_directory/" % self.current_path + new_dir = "%s/some_directory/" % self.current_path + os.makedirs(old_dir) + + time.sleep(5) + with pull(self.sh): + self.assert_new_commit() + + os.rename(old_dir, new_dir) + + time.sleep(5) + with pull(self.sh): + self.assert_new_commit() + + assert os.path.isdir(new_dir) is not False + assert os.path.exists(old_dir) is False + def test_link_a_file(self): filename = "%s/link_file" % self.current_path link_name = "%s/new_link" % self.current_path From f4a746d9b2c326b60588989758c3474ce3b6cd22 Mon Sep 17 00:00:00 2001 From: vtemian Date: Tue, 9 Dec 2014 17:54:24 +0200 Subject: [PATCH 09/11] Close #171: add tests for writing in .gitignore and .gitmodules --- tests/views/test_current.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/views/test_current.py b/tests/views/test_current.py index 993ebb74..c89f81b1 100644 --- a/tests/views/test_current.py +++ b/tests/views/test_current.py @@ -136,6 +136,22 @@ def test_write_in_git_dir(self): ignore=CachedIgnore()) current.write(".git/index", "buf", "offset", 1) + def test_write_in_gitignore_dir(self): + with pytest.raises(FuseOSError): + current = CurrentView(repo="repo", uid=1, gid=1, + repo_path="repo_path", + read_only=Event(), + ignore=CachedIgnore()) + current.write(".gitignore", "buf", "offset", 1) + + def test_write_in_modules_dir(self): + with pytest.raises(FuseOSError): + current = CurrentView(repo="repo", uid=1, gid=1, + repo_path="repo_path", + read_only=Event(), + ignore=CachedIgnore()) + current.write(".gitmodules", "buf", "offset", 1) + def test_write_to_large_file(self): current = CurrentView(repo="repo", uid=1, gid=1, repo_path="repo_path", From 95df7dae5d070587a6a4179dc4b655c399b127d1 Mon Sep 17 00:00:00 2001 From: vtemian Date: Tue, 9 Dec 2014 18:28:52 +0200 Subject: [PATCH 10/11] change the behaviour a little bit --- gitfs/cache/gitignore.py | 26 +++++++++++++++++--------- gitfs/mounter.py | 1 - gitfs/router.py | 13 +++++++++---- gitfs/utils/args.py | 7 +------ 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index b017d9bd..e658085a 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -19,11 +19,13 @@ class CachedIgnore(object): - def __init__(self, ignore=False, submodules=False, hard_ignore=None): + def __init__(self, ignore=False, submodules=False, exclude=False, + hard_ignore=None): self.items = [] self.ignore = ignore self.submodules = submodules + self.exclude = exclude self.cache = {} self.permanent = [] @@ -32,15 +34,10 @@ def __init__(self, ignore=False, submodules=False, hard_ignore=None): self.update() def update(self): - self.items = ['.git', '.git/*', '/.git/*', '*.keep', - '*.gitignore', '*.gitmodules'] + self.items = ['.git', '.git/*', '/.git/*', '*.keep', '*.gitmodules'] - if self.ignore and os.path.exists(self.ignore): - with open(self.ignore) as gitignore: - for item in gitignore.readlines(): - item = item.strip() - if item and not item.startswith('#'): - self.items.append(item) + self.items += self._parse_ignore_file(self.ignore) + self.items += self._parse_ignore_file(self.exclude) if self.submodules and os.path.exists(self.submodules): with open(self.submodules) as submodules: @@ -55,6 +52,17 @@ def update(self): self.cache = {} self.items += self.hard_ignore + def _parse_ignore_file(self, ignore_file): + items = [] + + if ignore_file and os.path.exists(ignore_file): + with open(ignore_file) as gitignore: + for item in gitignore.readlines(): + item = item.strip() + if item and not item.startswith('#'): + items.append(item) + return items + def _parse_hard_ignore(self, hard_ignore): if isinstance(hard_ignore, basestring): return hard_ignore.split("|") diff --git a/gitfs/mounter.py b/gitfs/mounter.py index a98b4ee5..dbef0df1 100644 --- a/gitfs/mounter.py +++ b/gitfs/mounter.py @@ -62,7 +62,6 @@ def prepare_components(args): max_offset=args.max_size * 1024 * 1024, commit_queue=commit_queue, credentials=credentials, - module_file=args.module_file, ignore_file=args.ignore_file, hard_ignore=args.hard_ignore) diff --git a/gitfs/router.py b/gitfs/router.py index 6c8ca0de..b94acf24 100644 --- a/gitfs/router.py +++ b/gitfs/router.py @@ -14,9 +14,10 @@ import re -import inspect -import shutil +import os import time +import shutil +import inspect from pwd import getpwnam from grp import getgrnam @@ -59,8 +60,12 @@ def __init__(self, remote_url, repo_path, mount_path, credentials, log.info('Done cloning') self.repo.credentials = credentials - self.repo.ignore = CachedIgnore(submodules=kwargs['module_file'], - ignore=kwargs['ignore_file'], + + submodules = os.path.join(self.repo_path, '.gitmodules') + ignore = os.path.join(self.repo_path, '.gitignore') + self.repo.ignore = CachedIgnore(submodules=submodules, + ignore=ignore, + exclude=kwargs['ignore_file'] or None, hard_ignore=kwargs['hard_ignore']) self.uid = getpwnam(user).pw_uid diff --git a/gitfs/utils/args.py b/gitfs/utils/args.py index 69d55fa4..9719096f 100644 --- a/gitfs/utils/args.py +++ b/gitfs/utils/args.py @@ -53,9 +53,7 @@ def __init__(self, parser): ("log_level", ("warning", "string")), ("cache_size", (800, "int")), ("sentry_dsn", (self.get_sentry_dsn, "string")), - ("ignore", ("", "string")), - ("ignore_file", (self.get_dot_file('.gitignore'), "string")), - ("module_file", (self.get_dot_file('.gitmodules'), "string")), + ("ignore_file", ("", "string")), ("hard_ignore", ("", "string")), ]) self.config = self.build_config(parser.parse_args()) @@ -165,9 +163,6 @@ def get_repo_path(self, args): def get_ssh_key(self, args): return os.environ["HOME"] + "/.ssh/id_rsa" - def get_dot_file(self, default_file): - return lambda args: os.path.join(args.repo_path, default_file) - def get_sentry_dsn(self, args): return os.environ["SENTRY_DSN"] if "SENTRY_DSN" in os.environ else "" From 4e34c383645b162ea8c523059cecd732223620ca Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 10 Dec 2014 12:27:31 +0200 Subject: [PATCH 11/11] fixed tests --- tests/cache/test_gitignore.py | 5 ++--- tests/test_router.py | 5 +++-- tests/views/test_current.py | 8 -------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/cache/test_gitignore.py b/tests/cache/test_gitignore.py index 3ac2243e..5e309d7a 100644 --- a/tests/cache/test_gitignore.py +++ b/tests/cache/test_gitignore.py @@ -33,9 +33,8 @@ def test_init(self): gitignore = CachedIgnore("some_file", "some_file") assert gitignore.items == ['.git', '.git/*', '/.git/*', - '*.keep', '*.gitignore', - '*.gitmodules', '/found/*', - '/found', 'found'] + '*.keep', '*.gitmodules', + '/found/*', '/found', 'found'] def test_update(self): gitignore = CachedIgnore() diff --git a/tests/test_router.py b/tests/test_router.py index 04304d6f..bb2cc43a 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -95,9 +95,10 @@ def test_constructor(self): mocks['branch'], mocks['credentials']) mocks['repository'].clone.assert_called_once_with(*asserted_call) mocks['ignore'].assert_called_once_with(**{ - 'ignore': '', + 'ignore': 'repository_path/.gitignore', + 'exclude': None, 'hard_ignore': None, - 'submodules': '', + 'submodules': 'repository_path/.gitmodules' }) mocks['getpwnam'].assert_called_once_with(mocks['user']) mocks['getgrnam'].assert_called_once_with(mocks['group']) diff --git a/tests/views/test_current.py b/tests/views/test_current.py index c89f81b1..349cca7d 100644 --- a/tests/views/test_current.py +++ b/tests/views/test_current.py @@ -136,14 +136,6 @@ def test_write_in_git_dir(self): ignore=CachedIgnore()) current.write(".git/index", "buf", "offset", 1) - def test_write_in_gitignore_dir(self): - with pytest.raises(FuseOSError): - current = CurrentView(repo="repo", uid=1, gid=1, - repo_path="repo_path", - read_only=Event(), - ignore=CachedIgnore()) - current.write(".gitignore", "buf", "offset", 1) - def test_write_in_modules_dir(self): with pytest.raises(FuseOSError): current = CurrentView(repo="repo", uid=1, gid=1,