From c955baf63e307dc7f9c8421bd7e64a1218d544c0 Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 17:14:12 +0200 Subject: [PATCH 01/13] cache: sanitize path when we check it --- gitfs/cache/gitignore.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index 0ff2cb35..6956fefe 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -67,6 +67,9 @@ def check_key(self, key): return False def _check_item_and_key(self, item, key): + if key.startswith("/"): + key = key[1:] + if item == key: return True From f10d79ba1523e67dc59efadee3fc8172025443af Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 17:15:30 +0200 Subject: [PATCH 02/13] repository: solve checkout bug --- gitfs/repository.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gitfs/repository.py b/gitfs/repository.py index 639ee731..1022d375 100644 --- a/gitfs/repository.py +++ b/gitfs/repository.py @@ -77,6 +77,9 @@ def diverge(self, upstream, branch): def checkout(self, ref, *args, **kwargs): result = self._repo.checkout(ref, *args, **kwargs) + # update ignore cache after a checkout + self.ignore.update() + status = self._repo.status() for path, status in status.iteritems(): # path is in current status, move on @@ -85,11 +88,12 @@ def checkout(self, ref, *args, **kwargs): # check if file exists or not if path not in self._repo.index: - os.unlink(self._full_path(path)) + if path not in self.ignore: + os.unlink(self._full_path(path)) continue # check files stats - stats = self.get_git_object_stat(path) + stats = self.get_git_object_default_stats(ref, path) current_stat = os.lstat(self._full_path(path)) if stats['st_mode'] != current_stat.st_mode: From f52aeb30159590cbd27f285dc49ce486bd46b6e2 Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 17:16:14 +0200 Subject: [PATCH 03/13] sync: log if merge fails --- gitfs/worker/sync.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/gitfs/worker/sync.py b/gitfs/worker/sync.py index c0daf755..c32f4018 100644 --- a/gitfs/worker/sync.py +++ b/gitfs/worker/sync.py @@ -67,10 +67,10 @@ def on_idle(self): """ if not syncing.is_set(): - log.debug("Set syncing event (%d pending writes)", writers) + log.debug("Set syncing event (%d pending writes)", writers.value) syncing.set() else: - log.debug("Idling (%d pending writes)", writers) + log.debug("Idling (%d pending writes)", writers.value) if writers == 0: if self.commits: @@ -97,8 +97,11 @@ def sync(self): if self.repository.behind: log.debug("I'm behind so I start merging") - self.merge() - need_to_push = True + try: + self.merge() + need_to_push = True + except: + log.exception("Merge failed") if need_to_push: try: From bee233c07c2a1c8cf2b88496e786623d37c79a78 Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 18:23:54 +0200 Subject: [PATCH 04/13] fix #159: configure sentry --- gitfs/router.py | 8 +++++++- gitfs/utils/args.py | 15 ++++++++++++++- gitfs/worker/fetch.py | 5 +++-- gitfs/worker/peasant.py | 10 ++++++++++ gitfs/worker/sync.py | 3 ++- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/gitfs/router.py b/gitfs/router.py index 39cab4fb..6fdca209 100644 --- a/gitfs/router.py +++ b/gitfs/router.py @@ -17,6 +17,7 @@ import inspect import shutil import time +import socket from pwd import getpwnam from grp import getgrnam @@ -126,7 +127,12 @@ def __call__(self, operation, *args): view.__class__.__name__)) raise FuseOSError(ENOSYS) - return getattr(view, operation)(*args) + try: + return getattr(view, operation)(*args) + except FuseOSError as e: + raise e + except Exception: + log.exception("[%s] A system call failed" % socket.gethostname()) def register(self, routes): for regex, view in routes: diff --git a/gitfs/utils/args.py b/gitfs/utils/args.py index ada6bd7e..933863fb 100644 --- a/gitfs/utils/args.py +++ b/gitfs/utils/args.py @@ -25,6 +25,9 @@ from collections import OrderedDict from urlparse import urlparse +from raven.conf import setup_logging +from raven.handlers.logging import SentryHandler + from gitfs.log import log from gitfs.cache import lru_cache @@ -52,6 +55,7 @@ def __init__(self, parser): ("log", ("syslog", "string")), ("log_level", ("warning", "string")), ("cache_size", (800, "int")), + ("sentry_dsn", (self.get_sentry_dsn, "string")), ]) self.config = self.build_config(parser.parse_args()) @@ -90,8 +94,14 @@ def check_args(self, args): '%(message)s'.format(mount_point=args.mount_point) handler.setFormatter(Formatter(fmt=logger_fmt)) + if args.sentry_dsn: + sentry_handler = SentryHandler(args.sentry_dsn) + sentry_handler.setLevel("ERROR") + setup_logging(sentry_handler) + log.addHandler(sentry_handler) + + handler.setLevel(args.log_level.upper()) log.addHandler(handler) - log.setLevel(args.log_level.upper()) # set cache size lru_cache.maxsize = args.cache_size @@ -151,6 +161,9 @@ def get_repo_path(self, args): def get_ssh_key(self, args): return os.environ["HOME"] + "/.ssh/id_rsa" + def get_sentry_dsn(self, args): + return os.environ["SENTRY_DSN"] if "SENTRY_DSN" in os.environ else "" + def get_ssh_user(self, args): url = args.remote_url parse_result = urlparse(url) diff --git a/gitfs/worker/fetch.py b/gitfs/worker/fetch.py index 5bea1cca..8d33a945 100644 --- a/gitfs/worker/fetch.py +++ b/gitfs/worker/fetch.py @@ -7,7 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. @@ -21,7 +22,7 @@ class FetchWorker(Peasant): name = 'FetchWorker' - def run(self): + def work(self): while True: fetch.wait(self.timeout) diff --git a/gitfs/worker/peasant.py b/gitfs/worker/peasant.py index 61764c5c..9c9a54f2 100644 --- a/gitfs/worker/peasant.py +++ b/gitfs/worker/peasant.py @@ -13,8 +13,11 @@ # limitations under the License. +import socket from threading import Thread +from gitfs.log import log + class Peasant(Thread): def __init__(self, *args, **kwargs): @@ -22,3 +25,10 @@ def __init__(self, *args, **kwargs): for name, value in kwargs.iteritems(): setattr(self, name, value) + + def run(self): + try: + self.work() + except: + log.exception("[%s] A worker is not feeling well" % + socket.gethostname()) diff --git a/gitfs/worker/sync.py b/gitfs/worker/sync.py index c32f4018..13b662aa 100644 --- a/gitfs/worker/sync.py +++ b/gitfs/worker/sync.py @@ -40,7 +40,7 @@ def __init__(self, author_name, author_email, commiter_name, self.strategy = strategy self.commits = [] - def run(self): + def work(self): while True: if shutting_down.is_set(): log.info("Stop sync worker") @@ -66,6 +66,7 @@ def on_idle(self): In this case we are safe to merge and push. """ + raise ValueError("ameno") if not syncing.is_set(): log.debug("Set syncing event (%d pending writes)", writers.value) syncing.set() From 0f57c6944b368c240ce3bc45edc7bb0065d18c3a Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 18:24:30 +0200 Subject: [PATCH 05/13] added raven for requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 1c5c52ee..7519420e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ fusepy==2.0.2 pygit2==0.21.4 atomiclong +raven==5.1.1 From 9449ef97a286e1f28e83afcdf7b8dba42759d443 Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 18:45:12 +0200 Subject: [PATCH 06/13] remove leftovers --- gitfs/worker/sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gitfs/worker/sync.py b/gitfs/worker/sync.py index 13b662aa..5b7c8d0b 100644 --- a/gitfs/worker/sync.py +++ b/gitfs/worker/sync.py @@ -66,7 +66,6 @@ def on_idle(self): In this case we are safe to merge and push. """ - raise ValueError("ameno") if not syncing.is_set(): log.debug("Set syncing event (%d pending writes)", writers.value) syncing.set() From 7778ea2297dd49e8a77d7fffca33b60e6a2cdcec Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 18:45:24 +0200 Subject: [PATCH 07/13] removed hard deps on raven --- gitfs/utils/args.py | 6 +++--- requirements.txt | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gitfs/utils/args.py b/gitfs/utils/args.py index 933863fb..6f458b15 100644 --- a/gitfs/utils/args.py +++ b/gitfs/utils/args.py @@ -25,9 +25,6 @@ from collections import OrderedDict from urlparse import urlparse -from raven.conf import setup_logging -from raven.handlers.logging import SentryHandler - from gitfs.log import log from gitfs.cache import lru_cache @@ -95,6 +92,9 @@ def check_args(self, args): handler.setFormatter(Formatter(fmt=logger_fmt)) if args.sentry_dsn: + from raven.conf import setup_logging + from raven.handlers.logging import SentryHandler + sentry_handler = SentryHandler(args.sentry_dsn) sentry_handler.setLevel("ERROR") setup_logging(sentry_handler) diff --git a/requirements.txt b/requirements.txt index 7519420e..1c5c52ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ fusepy==2.0.2 pygit2==0.21.4 atomiclong -raven==5.1.1 From 22c217a27a53a5340ab1acd3c7554faef6cc38f6 Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 19:26:07 +0200 Subject: [PATCH 08/13] tests: fix tests --- gitfs/cache/gitignore.py | 2 +- gitfs/router.py | 3 ++- gitfs/utils/args.py | 2 +- gitfs/worker/sync.py | 2 +- tests/cache/test_gitignore.py | 8 ++++---- tests/utils/test_args.py | 2 ++ tests/workers/test_fetch.py | 4 ++-- tests/workers/test_sync.py | 6 +++--- 8 files changed, 16 insertions(+), 13 deletions(-) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index 6956fefe..8c35c631 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -36,7 +36,7 @@ def __init__(self, ignore=False, submodules=False, path="."): self.update() def update(self): - self.items = ['/.git', '.git/*', '/.git/*', '*.keep'] + self.items = ['.git', '.git/*', '/.git/*', '*.keep'] if self.ignore and os.path.exists(self.ignore): with open(self.ignore) as gitignore: diff --git a/gitfs/router.py b/gitfs/router.py index 6fdca209..827ea2ab 100644 --- a/gitfs/router.py +++ b/gitfs/router.py @@ -131,8 +131,9 @@ def __call__(self, operation, *args): return getattr(view, operation)(*args) except FuseOSError as e: raise e - except Exception: + except Exception as exception: log.exception("[%s] A system call failed" % socket.gethostname()) + raise exception def register(self, routes): for regex, view in routes: diff --git a/gitfs/utils/args.py b/gitfs/utils/args.py index 6f458b15..02a520c3 100644 --- a/gitfs/utils/args.py +++ b/gitfs/utils/args.py @@ -91,7 +91,7 @@ def check_args(self, args): '%(message)s'.format(mount_point=args.mount_point) handler.setFormatter(Formatter(fmt=logger_fmt)) - if args.sentry_dsn: + if args.sentry_dsn != '': from raven.conf import setup_logging from raven.handlers.logging import SentryHandler diff --git a/gitfs/worker/sync.py b/gitfs/worker/sync.py index 5b7c8d0b..a9b110b2 100644 --- a/gitfs/worker/sync.py +++ b/gitfs/worker/sync.py @@ -72,7 +72,7 @@ def on_idle(self): else: log.debug("Idling (%d pending writes)", writers.value) - if writers == 0: + if writers.value == 0: if self.commits: log.info("Get some commits") self.commit(self.commits) diff --git a/tests/cache/test_gitignore.py b/tests/cache/test_gitignore.py index a8817bb5..6d2a6c27 100644 --- a/tests/cache/test_gitignore.py +++ b/tests/cache/test_gitignore.py @@ -32,9 +32,9 @@ def test_init(self): re=mocked_re): gitignore = CachedIgnore("some_file", "some_file") - assert gitignore.items == ['/.git', '.git/*', '/.git/*', '*.keep', - 'file', '/found/*', '/found', - 'found'] + assert gitignore.items == ['.git', '.git/*', '/.git/*', + '*.keep', 'file', '/found/*', + '/found', 'found'] def test_update(self): gitignore = CachedIgnore() @@ -47,5 +47,5 @@ def test_update(self): def test_contains(self): gitignore = CachedIgnore() - assert '/.git' in gitignore + assert '.git' in gitignore assert 'file' not in gitignore diff --git a/tests/utils/test_args.py b/tests/utils/test_args.py index 4f7dedad..61951956 100644 --- a/tests/utils/test_args.py +++ b/tests/utils/test_args.py @@ -35,6 +35,7 @@ def test_args(self): mocked_file.mkdtemp.return_value = "/tmp" mocked_pass.getuser.return_value = "test_user" mocked_os.getgid.return_value = 1 + mocked_os.environ = {} mocked_os.path.abspath.return_value = "abs/tmp" mocked_grp.getgrgid().gr_name = "test_group" mocked_parser.parse_args.return_value = mocked_args @@ -48,6 +49,7 @@ def test_args(self): mocked_args.user = None mocked_args.branch = None mocked_args.ssh_user = None + mocked_args.sentry_dsn = '' with patch.multiple('gitfs.utils.args', os=mocked_os, grp=mocked_grp, getpass=mocked_pass, tempfile=mocked_file, diff --git a/tests/workers/test_fetch.py b/tests/workers/test_fetch.py index 4028b981..a7d6902f 100644 --- a/tests/workers/test_fetch.py +++ b/tests/workers/test_fetch.py @@ -20,7 +20,7 @@ class TestFetchWorker(object): - def test_run(self): + def test_work(self): mocked_peasant = MagicMock() mocked_fetch = MagicMock(side_effect=ValueError) mocked_fetch_event = MagicMock() @@ -32,7 +32,7 @@ def test_run(self): worker.timeout = 5 with pytest.raises(ValueError): - worker.run() + worker.work() assert mocked_fetch.call_count == 1 mocked_fetch_event.wait.assert_called_once_with(5) diff --git a/tests/workers/test_sync.py b/tests/workers/test_sync.py index 2a8a47a7..463db3a9 100644 --- a/tests/workers/test_sync.py +++ b/tests/workers/test_sync.py @@ -22,7 +22,7 @@ class TestSyncWorker(object): - def test_run(self): + def test_work(self): mocked_queue = MagicMock() mocked_idle = MagicMock(side_effect=ValueError) @@ -34,7 +34,7 @@ def test_run(self): worker.timeout = 1 with pytest.raises(ValueError): - worker.run() + worker.work() mocked_queue.get.assert_called_once_with(timeout=1, block=True) assert mocked_idle.call_count == 1 @@ -47,7 +47,7 @@ def test_on_idle_with_commits_and_merges(self): mocked_syncing.is_set.return_value = False with patch.multiple("gitfs.worker.sync", syncing=mocked_syncing, - writers=0): + writers=MagicMock(value=0)): worker = SyncWorker("name", "email", "name", "email", strategy="strategy") worker.commits = "commits" From 8acd38d2c8681dc6b2ebcedb0aea25d3a4afe53c Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 19:28:35 +0200 Subject: [PATCH 09/13] when merge fails, put the fs in read-only --- gitfs/worker/sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gitfs/worker/sync.py b/gitfs/worker/sync.py index a9b110b2..4af010ed 100644 --- a/gitfs/worker/sync.py +++ b/gitfs/worker/sync.py @@ -102,6 +102,7 @@ def sync(self): need_to_push = True except: log.exception("Merge failed") + return if need_to_push: try: From e4e2f940c96417ddc36b2bb18cd014b3ebeffccc Mon Sep 17 00:00:00 2001 From: vtemian Date: Wed, 12 Nov 2014 19:43:47 +0200 Subject: [PATCH 10/13] remove socket.gethostname --- gitfs/router.py | 3 +-- gitfs/worker/peasant.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/gitfs/router.py b/gitfs/router.py index 827ea2ab..3e0ad9c8 100644 --- a/gitfs/router.py +++ b/gitfs/router.py @@ -17,7 +17,6 @@ import inspect import shutil import time -import socket from pwd import getpwnam from grp import getgrnam @@ -132,7 +131,7 @@ def __call__(self, operation, *args): except FuseOSError as e: raise e except Exception as exception: - log.exception("[%s] A system call failed" % socket.gethostname()) + log.exception("A system call failed") raise exception def register(self, routes): diff --git a/gitfs/worker/peasant.py b/gitfs/worker/peasant.py index 9c9a54f2..34defb1e 100644 --- a/gitfs/worker/peasant.py +++ b/gitfs/worker/peasant.py @@ -13,7 +13,6 @@ # limitations under the License. -import socket from threading import Thread from gitfs.log import log @@ -30,5 +29,4 @@ def run(self): try: self.work() except: - log.exception("[%s] A worker is not feeling well" % - socket.gethostname()) + log.exception("A worker is not feeling well") From 3c5a5cf21e736bc6f08aa534d4e683cb1ae3858c Mon Sep 17 00:00:00 2001 From: vtemian Date: Thu, 13 Nov 2014 17:57:57 +0200 Subject: [PATCH 11/13] gitignore: if the pattern endswith /, check if the current path, doesnt starts with pattern --- gitfs/cache/gitignore.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index 8c35c631..1e6c3270 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -73,4 +73,7 @@ def _check_item_and_key(self, item, key): if item == key: return True + if item.endswith("/"): + return key.startswith(item) + return fnmatch.fnmatch(key, item) From 95d5994b1d369a2bfc97dffd3555565e14e32dcc Mon Sep 17 00:00:00 2001 From: Calin Don Date: Thu, 13 Nov 2014 18:02:21 +0200 Subject: [PATCH 12/13] Handle line endings and comments in gitignore --- gitfs/cache/gitignore.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index 1e6c3270..21652480 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -40,10 +40,10 @@ def update(self): if self.ignore and os.path.exists(self.ignore): with open(self.ignore) as gitignore: - new_items = filter(lambda line: line != "", - gitignore.read().split("\n")) - - self.items += new_items + for item in gitignore.readlines(): + item = item.strip() + if item and not item.startswith('#'): + self.items += item if self.submodules and os.path.exists(self.submodules): with open(self.submodules) as submodules: From e351ab504610d30cedc212fb7e6151270234e3fe Mon Sep 17 00:00:00 2001 From: vtemian Date: Thu, 13 Nov 2014 18:05:23 +0200 Subject: [PATCH 13/13] gitignore: fix wrong condition --- gitfs/cache/gitignore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitfs/cache/gitignore.py b/gitfs/cache/gitignore.py index 1e6c3270..0a9d8b20 100644 --- a/gitfs/cache/gitignore.py +++ b/gitfs/cache/gitignore.py @@ -73,7 +73,7 @@ def _check_item_and_key(self, item, key): if item == key: return True - if item.endswith("/"): - return key.startswith(item) + if item.endswith("/") and key.startswith(item): + return True return fnmatch.fnmatch(key, item)