Implement opt-out user analytics

dvc/analytics.py

@@ -0,0 +1,211 @@
+import os
+import json
+import errno
+
+from dvc import VERSION
+from dvc.utils import is_binary
+from dvc.lock import Lock, LockError
+from dvc.logger import Logger
+
+
+class Analytics(object):
+    URL = 'https://dddc07bh9f.execute-api.us-east-2.amazonaws.com/production'
+    TIMEOUT_POST = 10
+
+    USER_ID_FILE = 'user_id'
+
+    PARAM_DVC_VERSION = 'dvc_version'
+    PARAM_USER_ID = 'user_id'
+    PARAM_SYSTEM_INFO = 'system_info'
+
+    PARAM_OS = 'os'
+
+    PARAM_WINDOWS_VERSION_MAJOR = 'windows_version_major'
+    PARAM_WINDOWS_VERSION_MINOR = 'windows_version_minor'
+    PARAM_WINDOWS_VERSION_BUILD = 'windows_version_build'
+    PARAM_WINDOWS_VERSION_SERVICE_PACK = 'windows_version_service_pack'
+
+    PARAM_MAC_VERSION = 'mac_version'
+
+    PARAM_LINUX_DISTRO = 'linux_distro'
+    PARAM_LINUX_DISTRO_VERSION = 'linux_distro_version'
+    PARAM_LINUX_DISTRO_LIKE = 'linux_distro_like'
+
+    PARAM_SCM_CLASS = 'scm_class'
+    PARAM_IS_BINARY = 'is_binary'
+    PARAM_CMD_CLASS = 'cmd_class'
+    PARAM_CMD_RETURN_CODE = 'cmd_return_code'
+
+    def __init__(self, info={}):
+        from dvc.config import Config
+
+        self.info = info
+
+        cdir = Config.get_global_config_dir()
+        try:
+            os.makedirs(cdir)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
+        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock')
+
+    @staticmethod
+    def load(path):
+        with open(path, 'r') as fobj:
+            a = Analytics(info=json.load(fobj))
+        os.unlink(path)
+        return a
+
+    def _write_user_id(self):
+        import uuid
+
+        with open(self.user_id_file, 'w+') as fobj:
+            user_id = str(uuid.uuid4())
+            info = {self.PARAM_USER_ID: user_id}
+            json.dump(info, fobj)
+            return user_id
+
+    def _read_user_id(self):
+        if not os.path.exists(self.user_id_file):
+            return None
+
+        with open(self.user_id_file, 'r') as fobj:
+            try:
+                info = json.load(fobj)
+                return info[self.PARAM_USER_ID]
+            except json.JSONDecodeError as exc:
+                Logger.debug("Failed to load user_id: {}".format(exc))
+                return None
+
+    def _get_user_id(self):
+        try:
+            with self.user_id_file_lock:
+                user_id = self._read_user_id()
+                if user_id is None:
+                    user_id = self._write_user_id()
+                return user_id
+        except LockError:
+            msg = "Failed to acquire '{}'"
+            Logger.debug(msg.format(self.user_id_file_lock.lock_file))
+
+    def _collect_windows(self):
+        import sys
+        version = sys.getwindowsversion()
+        info = {}
+        info[self.PARAM_OS] = 'windows'
+        info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
+        info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
+        info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
+        info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack
+        return info
+
+    def _collect_darwin(self):
+        import platform
+        info = {}
+        info[self.PARAM_OS] = 'mac'
+        info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
+        return info
+
+    def _collect_linux(self):
+        import distro
+        info = {}
+        info[self.PARAM_OS] = 'linux'
+        info[self.PARAM_LINUX_DISTRO] = distro.id()
+        info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
+        info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
+        return info
+
+    def _collect_system_info(self):
+        import platform
+        system = platform.system()
+        if system == 'Windows':
+            return self._collect_windows()
+        elif system == 'Darwin':
+            return self._collect_darwin()
+        elif system == 'Linux':
+            return self._collect_linux()
+        else:
+            raise NotImplementedError
+
+    def _collect(self, args=None, ret=None):
+        from dvc.scm import SCM
+        from dvc.project import Project, NotDvcProjectError
+        from dvc.command.daemon import CmdDaemonAnalytics
+
+        self.info[self.PARAM_DVC_VERSION] = VERSION
+        self.info[self.PARAM_IS_BINARY] = is_binary()
+        self.info[self.PARAM_USER_ID] = self._get_user_id()
+
+        self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info()
+
+        try:
+            scm = SCM(root_dir=Project._find_root())
+            self.info[self.PARAM_SCM_CLASS] = type(scm).__name__
+        except NotDvcProjectError:
+            pass
+
+        if ret is not None:
+            self.info[self.PARAM_CMD_RETURN_CODE] = ret
+
+        if args is not None and hasattr(args, 'func'):
+            assert args.func != CmdDaemonAnalytics
+            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
+
+    def dump(self):
+        import tempfile
+
+        with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj:
+            json.dump(self.info, fobj)
+            return fobj.name
+
+    @staticmethod
+    def _is_enabled(cmd):
+        from dvc.config import Config
+        from dvc.project import Project, NotDvcProjectError
+        from dvc.command.daemon import CmdDaemonBase
+
+        if os.getenv('DVC_TEST'):
+            return False
+
+        if isinstance(cmd, CmdDaemonBase):
+            return False
+
+        if cmd is None or not hasattr(cmd, 'config'):
+            try:
+                dvc_dir = Project._find_dvc_dir()
+                config = Config(dvc_dir)
+                assert config is not None
+            except NotDvcProjectError:
+                config = Config(validate=False)
+                assert config is not None
+        else:
+            config = cmd.config
+            assert config is not None
+
+        core = config._config.get(Config.SECTION_CORE, {})
+        enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
+        Logger.debug("Analytics is {}abled."
+                     .format('en' if enabled else 'dis'))
+        return enabled
+
+    def send(self, detach=True, cmd=None, args=None, ret=None):
+        import requests
+        from dvc.daemon import Daemon
+
+        if not self._is_enabled(cmd):
+            return
+
+        if detach:  # pragma: no cover
+            Daemon()(['analytics', self.dump()])
+            return
+
+        self._collect(args=args, ret=ret)
+
+        Logger.debug("Sending analytics: {}".format(self.info))
+
+        try:
+            requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
+        except requests.exceptions.RequestException as exc:
+            Logger.debug("Failed to send analytics: {}".format(str(exc)))

dvc/cli.py

@@ -26,7 +26,8 @@
 from dvc.command.root import CmdRoot
 from dvc.command.lock import CmdLock, CmdUnlock
 from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList
-from dvc.command.daemon import CmdDaemonUpdater
+from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics
+from dvc.exceptions import DvcException
 from dvc.logger import Logger
 from dvc import VERSION
 
@@ -41,11 +42,16 @@ def _fix_subparsers(subparsers):
         subparsers.dest = 'cmd'
 
 
+class DvcParserError(DvcException):
+    def __init__(self):
+        super(DvcException, self).__init__("Parser error")
+
+
 class DvcParser(argparse.ArgumentParser):
     def error(self, message):
         sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message))
         self.print_help()
-        sys.exit(2)
+        raise DvcParserError()
 
 
 class VersionAction(argparse.Action):  # pragma: no cover
@@ -638,6 +644,17 @@ def parse_args(argv=None):
                         nargs='?',
                         default=None,
                         help='Option value.')
+    config_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    config_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     config_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -672,6 +689,17 @@ def parse_args(argv=None):
     remote_add_parser.add_argument(
                         'url',
                         help='URL.')
+    remote_add_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_add_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     remote_add_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -694,6 +722,17 @@ def parse_args(argv=None):
     remote_remove_parser.add_argument(
                         'name',
                         help='Name')
+    remote_remove_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_remove_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     remote_remove_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -723,6 +762,17 @@ def parse_args(argv=None):
                         default=False,
                         action='store_true',
                         help='Unset option.')
+    remote_modify_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_modify_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     remote_modify_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -736,6 +786,17 @@ def parse_args(argv=None):
                         parents=[parent_parser],
                         description=REMOTE_LIST_HELP,
                         help=REMOTE_LIST_HELP)
+    remote_list_parser.add_argument(
+                        '--global',
+                        dest='glob',
+                        action='store_true',
+                        default=False,
+                        help='Use global config.')
+    remote_list_parser.add_argument(
+                        '--system',
+                        action='store_true',
+                        default=False,
+                        help='Use system config.')
     remote_list_parser.add_argument(
                         '--local',
                         action='store_true',
@@ -982,6 +1043,17 @@ def parse_args(argv=None):
                         help=DAEMON_UPDATER_HELP)
     daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
 
+    DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.'
+    daemon_analytics_parser = daemon_subparsers.add_parser(
+                        'analytics',
+                        parents=[parent_parser],
+                        description=DAEMON_ANALYTICS_HELP,
+                        help=DAEMON_ANALYTICS_HELP)
+    daemon_analytics_parser.add_argument(
+                        'target',
+                        help="Analytics file.")
+    daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)
+
     args = parser.parse_args(argv)
 
     if (issubclass(args.func, CmdRepro)

dvc/command/base.py

@@ -1,15 +1,13 @@
-import os
-
 from dvc.logger import Logger
-from dvc.exceptions import DvcException
 from dvc.lock import LockError
 
 
 class CmdBase(object):
     def __init__(self, args):
         from dvc.project import Project
 
-        self.project = Project(self._find_root())
+        self.project = Project()
+        self.config = self.project.config
         self.args = args
         self._set_loglevel(args)
 
@@ -21,20 +19,6 @@ def _set_loglevel(args):
         elif args.verbose:
             Logger.be_verbose()
 
-    def _find_root(self):
-        from dvc.project import Project
-
-        root = os.getcwd()
-        while True:
-            dvc_dir = os.path.join(root, Project.DVC_DIR)
-            if os.path.isdir(dvc_dir):
-                return root
-            if os.path.ismount(root):
-                break
-            root = os.path.dirname(root)
-        msg = "Not a dvc repository (checked up to mount point {})"
-        raise DvcException(msg.format(root))
-
     def run_cmd(self):
         try:
             with self.project.lock: