Merge pull request #1337 from efiop/master

updater: spawn a background daemon

dvc/cli.py

@@ -26,6 +26,7 @@
 from dvc.command.root import CmdRoot
 from dvc.command.lock import CmdLock, CmdUnlock
 from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList
+from dvc.command.daemon import CmdDaemonUpdater
 from dvc.logger import Logger
 from dvc import VERSION
 
@@ -942,6 +943,29 @@ def parse_args(argv=None):
                         help=PIPELINE_LIST_HELP)
     pipeline_list_parser.set_defaults(func=CmdPipelineList)
 
+    # Daemon
+    DAEMON_HELP = 'Service daemon.'
+    daemon_parser = subparsers.add_parser(
+                        'daemon',
+                        parents=[parent_parser],
+                        description=DAEMON_HELP,
+                        help=DAEMON_HELP)
+
+    daemon_subparsers = daemon_parser.add_subparsers(
+                        dest='cmd',
+                        help='Use dvc daemon CMD --help '
+                             'for command-specific help.')
+
+    _fix_subparsers(daemon_subparsers)
+
+    DAEMON_UPDATER_HELP = 'Fetch latest available version.'
+    daemon_updater_parser = daemon_subparsers.add_parser(
+                        'updater',
+                        parents=[parent_parser],
+                        description=DAEMON_UPDATER_HELP,
+                        help=DAEMON_UPDATER_HELP)
+    daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
+
     args = parser.parse_args(argv)
 
     if (issubclass(args.func, CmdRepro)

dvc/command/daemon.py

@@ -0,0 +1,19 @@
+from dvc.command.base import CmdBase
+
+
+class CmdDaemonUpdater(CmdBase):
+    def __init__(self, args):
+        pass
+
+    def run_cmd(self):
+        return self.run()
+
+    def run(self):
+        import os
+        from dvc.project import Project
+        from dvc.updater import Updater
+
+        root_dir = self._find_root()
+        dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
+        updater = Updater(dvc_dir)
+        updater.fetch(detach=False)

dvc/daemon.py

@@ -0,0 +1,62 @@
+import os
+import sys
+from subprocess import Popen
+from dvc.logger import Logger
+
+
+class Daemon(object):  # pragma: no cover
+    def _spawn_windows(self, cmd):
+        CREATE_NEW_PROCESS_GROUP = 0x00000200
+        DETACHED_PROCESS = 0x00000008
+        creationflags = CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
+        Popen(cmd,
+              close_fds=True,
+              shell=False,
+              creationflags=creationflags)
+
+    def _spawn_posix(self, cmd):
+        try:
+            pid = os.fork()
+            if pid > 0:
+                return
+        except OSError as exc:
+            Logger.error("Failed at first fork", exc)
+            sys.exit(1)
+
+        os.setsid()
+        os.umask(0)
+
+        try:
+            pid = os.fork()
+            if pid > 0:
+                sys.exit(0)
+        except OSError as exc:
+            Logger.error("Failed at second fork", exc)
+            sys.exit(1)
+
+        sys.stdin.close()
+        sys.stdout.close()
+        sys.stderr.close()
+
+        Popen(cmd,
+              close_fds=True,
+              shell=False)
+
+    def __call__(self, name):
+        from dvc.utils import is_binary
+
+        cmd = [sys.executable]
+        if not is_binary():
+            cmd += ['-m', 'dvc']
+        cmd += ['daemon', name, '-v']
+
+        Logger.debug("Trying to spawn '{}'".format(cmd))
+
+        if os.name == 'nt':
+            self._spawn_windows(cmd)
+        elif os.name == 'posix':
+            self._spawn_posix(cmd)
+        else:
+            raise NotImplementedError
+
+        Logger.debug("Spawned '{}'".format(cmd))

dvc/project.py

@@ -136,6 +136,7 @@ def _ignore(self):
             self.lock.lock_file,
             self.config.config_local_file,
             self.updater.updater_file,
+            self.updater.lock.lock_file,
         ] + self.state.temp_files
 
         if self.cache.local.cache_dir.startswith(self.root_dir):

dvc/updater.py

@@ -6,6 +6,7 @@
 
 from dvc import VERSION_BASE
 from dvc.logger import Logger
+from dvc.utils import is_binary
 
 
 class Updater(object):  # pragma: no cover
@@ -16,40 +17,65 @@ class Updater(object):  # pragma: no cover
     TIMEOUT_GET = 10
 
     def __init__(self, dvc_dir):
+        from dvc.lock import Lock
+
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
+        self.lock = Lock(dvc_dir, self.updater_file + '.lock')
         self.current = VERSION_BASE
 
+    def _is_outdated_file(self):
+        ctime = os.path.getmtime(self.updater_file)
+        outdated = (time.time() - ctime >= self.TIMEOUT)
+        if outdated:
+            Logger.debug("'{}' is outdated(".format(self.updater_file))
+        return outdated
+
     def check(self):
         if os.getenv('CI'):
             return
 
-        if os.path.isfile(self.updater_file):
-            ctime = os.path.getctime(self.updater_file)
-            if time.time() - ctime < self.TIMEOUT:
-                msg = '{} is not old enough to check for updates'
-                Logger.debug(msg.format(self.UPDATER_FILE))
-                return
+        with self.lock:
+            self._check()
 
-            os.unlink(self.updater_file)
+    def _check(self):
+        if not os.path.exists(self.updater_file) or self._is_outdated_file():
+            self.fetch()
+            return
 
-        Logger.info('Checking for updates...')
+        with open(self.updater_file, 'r') as fobj:
+            import json
 
-        try:
-            self._get_latest_version()
-        except Exception as exc:
-            msg = 'Failed to obtain latest version: {}'.format(str(exc))
-            Logger.debug(msg)
-            return
+            try:
+                info = json.load(fobj)
+                self.latest = info['version']
+            except json.decoder.JSONDecodeError as exc:
+                msg = "'{}' is not a valid json: {}"
+                Logger.debug(msg.format(self.updater_file, exc))
+                self.fetch()
+                return
 
         if self._is_outdated():
             self._notify()
 
+    def fetch(self, detach=True):
+        from dvc.daemon import Daemon
+
+        if detach:
+            d = Daemon()
+            d('updater')
+            return
+
+        with self.lock:
+            self._get_latest_version()
+
     def _get_latest_version(self):
+        import json
+
         r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
-        j = r.json()
-        self.latest = j['version']
-        open(self.updater_file, 'w+').close()
+        info = r.json()
+        with open(self.updater_file, 'w+') as fobj:
+            json.dump(info, fobj)
 
     def _is_outdated(self):
         l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')]
@@ -97,14 +123,10 @@ def _get_update_instructions(self):
 
         return instructions[package_manager]
 
-    @staticmethod
-    def _is_binary():
-        return getattr(sys, 'frozen', False)
-
     def _get_linux(self):
         import distro
 
-        if not self._is_binary():
+        if not is_binary():
             return 'pip'
 
         package_managers = {
@@ -120,7 +142,7 @@ def _get_linux(self):
         return package_managers.get(distro.id())
 
     def _get_darwin(self):
-        if not self._is_binary():
+        if not is_binary():
             if __file__.startswith('/usr/local/Cellar'):
                 return 'formula'
             else:
@@ -136,7 +158,7 @@ def _get_darwin(self):
         return None
 
     def _get_windows(self):
-        return None if self._is_binary() else 'pip'
+        return None if is_binary() else 'pip'
 
     def _get_package_manager(self):
         import platform

dvc/utils.py

@@ -146,6 +146,12 @@ def to_chunks(l, jobs):
     return [l[x:x+n] for x in range(0, len(l), n)]
 
 
+# NOTE: Check if we are in a bundle
+# https://pythonhosted.org/PyInstaller/runtime-information.html
+def is_binary():
+    return getattr(sys, 'frozen', False)
+
+
 # NOTE: Fix env variables modified by PyInstaller
 # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
 def fix_env(env):
@@ -154,9 +160,7 @@ def fix_env(env):
     else:
         env = env.copy()
 
-    # NOTE: Check if we are in a bundle
-    # https://pythonhosted.org/PyInstaller/runtime-information.html
-    if getattr(sys, 'frozen', False):
+    if is_binary():
         lp_key = 'LD_LIBRARY_PATH'
         lp_orig = env.get(lp_key + '_ORIG', None)
         if lp_orig is not None: