diff --git a/dvc/analytics.py b/dvc/analytics.py new file mode 100644 index 0000000000..06bcb3bef4 --- /dev/null +++ b/dvc/analytics.py @@ -0,0 +1,211 @@ +import os +import json +import errno + +from dvc import VERSION +from dvc.utils import is_binary +from dvc.lock import Lock, LockError +from dvc.logger import Logger + + +class Analytics(object): + URL = 'https://dddc07bh9f.execute-api.us-east-2.amazonaws.com/production' + TIMEOUT_POST = 10 + + USER_ID_FILE = 'user_id' + + PARAM_DVC_VERSION = 'dvc_version' + PARAM_USER_ID = 'user_id' + PARAM_SYSTEM_INFO = 'system_info' + + PARAM_OS = 'os' + + PARAM_WINDOWS_VERSION_MAJOR = 'windows_version_major' + PARAM_WINDOWS_VERSION_MINOR = 'windows_version_minor' + PARAM_WINDOWS_VERSION_BUILD = 'windows_version_build' + PARAM_WINDOWS_VERSION_SERVICE_PACK = 'windows_version_service_pack' + + PARAM_MAC_VERSION = 'mac_version' + + PARAM_LINUX_DISTRO = 'linux_distro' + PARAM_LINUX_DISTRO_VERSION = 'linux_distro_version' + PARAM_LINUX_DISTRO_LIKE = 'linux_distro_like' + + PARAM_SCM_CLASS = 'scm_class' + PARAM_IS_BINARY = 'is_binary' + PARAM_CMD_CLASS = 'cmd_class' + PARAM_CMD_RETURN_CODE = 'cmd_return_code' + + def __init__(self, info={}): + from dvc.config import Config + + self.info = info + + cdir = Config.get_global_config_dir() + try: + os.makedirs(cdir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) + self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock') + + @staticmethod + def load(path): + with open(path, 'r') as fobj: + a = Analytics(info=json.load(fobj)) + os.unlink(path) + return a + + def _write_user_id(self): + import uuid + + with open(self.user_id_file, 'w+') as fobj: + user_id = str(uuid.uuid4()) + info = {self.PARAM_USER_ID: user_id} + json.dump(info, fobj) + return user_id + + def _read_user_id(self): + if not os.path.exists(self.user_id_file): + return None + + with open(self.user_id_file, 'r') as fobj: + try: + info = json.load(fobj) + return info[self.PARAM_USER_ID] + except json.JSONDecodeError as exc: + Logger.debug("Failed to load user_id: {}".format(exc)) + return None + + def _get_user_id(self): + try: + with self.user_id_file_lock: + user_id = self._read_user_id() + if user_id is None: + user_id = self._write_user_id() + return user_id + except LockError: + msg = "Failed to acquire '{}'" + Logger.debug(msg.format(self.user_id_file_lock.lock_file)) + + def _collect_windows(self): + import sys + version = sys.getwindowsversion() + info = {} + info[self.PARAM_OS] = 'windows' + info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major + info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor + info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build + info[self.PARAM_WINDOWS_VERSION_SERVICE_PACK] = version.service_pack + return info + + def _collect_darwin(self): + import platform + info = {} + info[self.PARAM_OS] = 'mac' + info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0] + return info + + def _collect_linux(self): + import distro + info = {} + info[self.PARAM_OS] = 'linux' + info[self.PARAM_LINUX_DISTRO] = distro.id() + info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version() + info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like() + return info + + def _collect_system_info(self): + import platform + system = platform.system() + if system == 'Windows': + return self._collect_windows() + elif system == 'Darwin': + return self._collect_darwin() + elif system == 'Linux': + return self._collect_linux() + else: + raise NotImplementedError + + def _collect(self, args=None, ret=None): + from dvc.scm import SCM + from dvc.project import Project, NotDvcProjectError + from dvc.command.daemon import CmdDaemonAnalytics + + self.info[self.PARAM_DVC_VERSION] = VERSION + self.info[self.PARAM_IS_BINARY] = is_binary() + self.info[self.PARAM_USER_ID] = self._get_user_id() + + self.info[self.PARAM_SYSTEM_INFO] = self._collect_system_info() + + try: + scm = SCM(root_dir=Project._find_root()) + self.info[self.PARAM_SCM_CLASS] = type(scm).__name__ + except NotDvcProjectError: + pass + + if ret is not None: + self.info[self.PARAM_CMD_RETURN_CODE] = ret + + if args is not None and hasattr(args, 'func'): + assert args.func != CmdDaemonAnalytics + self.info[self.PARAM_CMD_CLASS] = args.func.__name__ + + def dump(self): + import tempfile + + with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj: + json.dump(self.info, fobj) + return fobj.name + + @staticmethod + def _is_enabled(cmd): + from dvc.config import Config + from dvc.project import Project, NotDvcProjectError + from dvc.command.daemon import CmdDaemonBase + + if os.getenv('DVC_TEST'): + return False + + if isinstance(cmd, CmdDaemonBase): + return False + + if cmd is None or not hasattr(cmd, 'config'): + try: + dvc_dir = Project._find_dvc_dir() + config = Config(dvc_dir) + assert config is not None + except NotDvcProjectError: + config = Config(validate=False) + assert config is not None + else: + config = cmd.config + assert config is not None + + core = config._config.get(Config.SECTION_CORE, {}) + enabled = core.get(Config.SECTION_CORE_ANALYTICS, True) + Logger.debug("Analytics is {}abled." + .format('en' if enabled else 'dis')) + return enabled + + def send(self, detach=True, cmd=None, args=None, ret=None): + import requests + from dvc.daemon import Daemon + + if not self._is_enabled(cmd): + return + + if detach: # pragma: no cover + Daemon()(['analytics', self.dump()]) + return + + self._collect(args=args, ret=ret) + + Logger.debug("Sending analytics: {}".format(self.info)) + + try: + requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST) + except requests.exceptions.RequestException as exc: + Logger.debug("Failed to send analytics: {}".format(str(exc))) diff --git a/dvc/cli.py b/dvc/cli.py index 0c3a5eaa18..607911670d 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -26,7 +26,8 @@ from dvc.command.root import CmdRoot from dvc.command.lock import CmdLock, CmdUnlock from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList -from dvc.command.daemon import CmdDaemonUpdater +from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics +from dvc.exceptions import DvcException from dvc.logger import Logger from dvc import VERSION @@ -41,11 +42,16 @@ def _fix_subparsers(subparsers): subparsers.dest = 'cmd' +class DvcParserError(DvcException): + def __init__(self): + super(DvcException, self).__init__("Parser error") + + class DvcParser(argparse.ArgumentParser): def error(self, message): sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message)) self.print_help() - sys.exit(2) + raise DvcParserError() class VersionAction(argparse.Action): # pragma: no cover @@ -638,6 +644,17 @@ def parse_args(argv=None): nargs='?', default=None, help='Option value.') + config_parser.add_argument( + '--global', + dest='glob', + action='store_true', + default=False, + help='Use global config.') + config_parser.add_argument( + '--system', + action='store_true', + default=False, + help='Use system config.') config_parser.add_argument( '--local', action='store_true', @@ -672,6 +689,17 @@ def parse_args(argv=None): remote_add_parser.add_argument( 'url', help='URL.') + remote_add_parser.add_argument( + '--global', + dest='glob', + action='store_true', + default=False, + help='Use global config.') + remote_add_parser.add_argument( + '--system', + action='store_true', + default=False, + help='Use system config.') remote_add_parser.add_argument( '--local', action='store_true', @@ -694,6 +722,17 @@ def parse_args(argv=None): remote_remove_parser.add_argument( 'name', help='Name') + remote_remove_parser.add_argument( + '--global', + dest='glob', + action='store_true', + default=False, + help='Use global config.') + remote_remove_parser.add_argument( + '--system', + action='store_true', + default=False, + help='Use system config.') remote_remove_parser.add_argument( '--local', action='store_true', @@ -723,6 +762,17 @@ def parse_args(argv=None): default=False, action='store_true', help='Unset option.') + remote_modify_parser.add_argument( + '--global', + dest='glob', + action='store_true', + default=False, + help='Use global config.') + remote_modify_parser.add_argument( + '--system', + action='store_true', + default=False, + help='Use system config.') remote_modify_parser.add_argument( '--local', action='store_true', @@ -736,6 +786,17 @@ def parse_args(argv=None): parents=[parent_parser], description=REMOTE_LIST_HELP, help=REMOTE_LIST_HELP) + remote_list_parser.add_argument( + '--global', + dest='glob', + action='store_true', + default=False, + help='Use global config.') + remote_list_parser.add_argument( + '--system', + action='store_true', + default=False, + help='Use system config.') remote_list_parser.add_argument( '--local', action='store_true', @@ -982,6 +1043,17 @@ def parse_args(argv=None): help=DAEMON_UPDATER_HELP) daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) + DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.' + daemon_analytics_parser = daemon_subparsers.add_parser( + 'analytics', + parents=[parent_parser], + description=DAEMON_ANALYTICS_HELP, + help=DAEMON_ANALYTICS_HELP) + daemon_analytics_parser.add_argument( + 'target', + help="Analytics file.") + daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) + args = parser.parse_args(argv) if (issubclass(args.func, CmdRepro) diff --git a/dvc/command/base.py b/dvc/command/base.py index 2a86c058b2..8befa169ee 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -1,7 +1,4 @@ -import os - from dvc.logger import Logger -from dvc.exceptions import DvcException from dvc.lock import LockError @@ -9,7 +6,8 @@ class CmdBase(object): def __init__(self, args): from dvc.project import Project - self.project = Project(self._find_root()) + self.project = Project() + self.config = self.project.config self.args = args self._set_loglevel(args) @@ -21,20 +19,6 @@ def _set_loglevel(args): elif args.verbose: Logger.be_verbose() - def _find_root(self): - from dvc.project import Project - - root = os.getcwd() - while True: - dvc_dir = os.path.join(root, Project.DVC_DIR) - if os.path.isdir(dvc_dir): - return root - if os.path.ismount(root): - break - root = os.path.dirname(root) - msg = "Not a dvc repository (checked up to mount point {})" - raise DvcException(msg.format(root)) - def run_cmd(self): try: with self.project.lock: diff --git a/dvc/command/config.py b/dvc/command/config.py index bd7c5b2041..75fb009fde 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,102 +1,79 @@ import os -import configobj from dvc.command.base import CmdBase from dvc.logger import Logger from dvc.config import Config +from dvc.exceptions import DvcException class CmdConfig(CmdBase): def __init__(self, args): - from dvc.project import Project + from dvc.project import Project, NotDvcProjectError self.args = args - root_dir = self._find_root() - if args.local: - config = Config.CONFIG_LOCAL + + try: + dvc_dir = os.path.join(Project._find_root(), Project.DVC_DIR) + saved_exc = None + except NotDvcProjectError as exc: + dvc_dir = None + saved_exc = exc + + self.config = Config(dvc_dir, validate=False) + if self.args.system: + self.configobj = self.config._system_config + elif self.args.glob: + self.configobj = self.config._global_config + elif self.args.local: + if dvc_dir is None: + raise saved_exc + self.configobj = self.config._local_config else: - config = Config.CONFIG - self.config_file = os.path.join(root_dir, Project.DVC_DIR, config) - # Using configobj because it doesn't - # drop comments like configparser does. - self.configobj = configobj.ConfigObj(self.config_file) + if dvc_dir is None: + raise saved_exc + self.configobj = self.config._project_config def run_cmd(self): return self.run() - def _get_key(self, d, name, add=False): - for k in d.keys(): - if k.lower() == name.lower(): - return k - - if add: - d[name] = {} - return name + def _unset(self, section, opt=None, configobj=None): + if configobj is None: + configobj = self.configobj - return None - - def save(self): try: - self.configobj.write() - except Exception as exc: - msg = "Failed to write config '{}'".format(self.configobj.filename) - Logger.error(msg, exc) + self.config.unset(configobj, section, opt) + self.config.save(configobj) + except DvcException as exc: + Logger.error("Failed to unset '{}'".format(self.args.name), exc) return 1 return 0 - def unset(self, section, opt=None): - if section not in self.configobj.keys(): - Logger.error("Section '{}' doesn't exist".format(section)) + def _show(self, section, opt): + try: + self.config.show(self.configobj, section, opt) + except DvcException as exc: + Logger.error("Failed to show '{}'".format(self.args.name), exc) return 1 - - if opt in self.configobj[section].keys(): - del self.configobj[section][opt] - - if len(self.configobj[section]) == 0 or opt is None: - del self.configobj[section] - - return self.save() - - def show(self): - Logger.info(self.configobj[self.section][self.opt]) return 0 - def set(self, section, opt, value): - if section not in self.configobj.keys(): - self.configobj[section] = {} - - self.configobj[section][opt] = value - - return self.save() - - def check_opt(self): - _section, _opt = self.args.name.strip().split('.', 1) - add = (self.args.value is not None and self.args.unset is False) - - section = self._get_key(self.configobj, _section, add) - - if not section: - Logger.error('Invalid option name {}'.format(_section)) - return 1 - - opt = self._get_key(self.configobj[section], _opt, add) - if not opt: - Logger.error('Invalid option value: {}'.format(_opt)) + def _set(self, section, opt, value): + try: + self.config.set(self.configobj, section, opt, value) + self.config.save(self.configobj) + except DvcException as exc: + Logger.error("Failed to set '{}.{}' to '{}'".format(section, + opt, + value), + exc) return 1 - - self.section = section - self.opt = opt - return 0 def run(self): - if self.check_opt() != 0: - return 1 + section, opt = self.args.name.lower().strip().split('.', 1) if self.args.unset: - return self.unset(self.section, self.opt) - - if self.args.value is None: - return self.show() - - return self.set(self.section, self.opt, self.args.value) + return self._unset(section, opt) + elif self.args.value is None: + return self._show(section, opt) + else: + return self._set(section, opt, self.args.value) diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 82ca1bf0c4..1e4a023804 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -1,19 +1,35 @@ from dvc.command.base import CmdBase -class CmdDaemonUpdater(CmdBase): +class CmdDaemonBase(CmdBase): def __init__(self, args): - pass + self.args = args + self.config = None + self._set_loglevel(args) def run_cmd(self): return self.run() + +class CmdDaemonUpdater(CmdDaemonBase): def run(self): import os from dvc.project import Project from dvc.updater import Updater - root_dir = self._find_root() + root_dir = Project._find_root() dvc_dir = os.path.join(root_dir, Project.DVC_DIR) updater = Updater(dvc_dir) updater.fetch(detach=False) + + return 0 + + +class CmdDaemonAnalytics(CmdDaemonBase): + def run(self): + from dvc.analytics import Analytics + + analytics = Analytics.load(self.args.target) + analytics.send(detach=False) + + return 0 diff --git a/dvc/command/init.py b/dvc/command/init.py index ea6105dd3f..988551b324 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -9,7 +9,10 @@ def run_cmd(self): from dvc.project import Project, InitError try: - Project.init('.', no_scm=self.args.no_scm, force=self.args.force) + self.project = Project.init('.', + no_scm=self.args.no_scm, + force=self.args.force) + self.config = self.project.config except InitError as e: Logger.error('Failed to initiate dvc', e) return 1 diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 08a5b43348..384197285b 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -1,6 +1,4 @@ -import os import re -import configobj from dvc.config import Config from dvc.command.config import CmdConfig @@ -10,49 +8,50 @@ class CmdRemoteAdd(CmdConfig): def run(self): section = Config.SECTION_REMOTE_FMT.format(self.args.name) - ret = self.set(section, Config.SECTION_REMOTE_URL, self.args.url) + ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url) if ret != 0: return ret if self.args.default: msg = 'Setting \'{}\' as a default remote.'.format(self.args.name) Logger.info(msg) - ret = self.set(Config.SECTION_CORE, - Config.SECTION_CORE_REMOTE, - self.args.name) + ret = self._set(Config.SECTION_CORE, + Config.SECTION_CORE_REMOTE, + self.args.name) return ret class CmdRemoteRemove(CmdConfig): - def _remove_default(self, config_file, remote): - path = os.path.join(os.path.dirname(self.config_file), - config_file) - config = configobj.ConfigObj(path) - + def _remove_default(self, config): core = config.get(Config.SECTION_CORE, None) if core is None: - return + return 0 default = core.get(Config.SECTION_CORE_REMOTE, None) if default is None: - return - - if default == remote: - del config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE] - if len(config[Config.SECTION_CORE]) == 0: - del config[Config.SECTION_CORE] + return 0 - config.write() + if default == self.args.name: + return self._unset(Config.SECTION_CORE, + opt=Config.SECTION_CORE_REMOTE, + configobj=config) def run(self): section = Config.SECTION_REMOTE_FMT.format(self.args.name) - ret = self.unset(section) + ret = self._unset(section) if ret != 0: return ret - self._remove_default(Config.CONFIG, self.args.name) - self._remove_default(Config.CONFIG_LOCAL, self.args.name) + for configobj in [self.config._local_config, + self.config._project_config, + self.config._global_config, + self.config._system_config]: + self._remove_default(configobj) + self.config.save(configobj) + if configobj == self.configobj: + break + return 0 diff --git a/dvc/config.py b/dvc/config.py index 8b8af2fb12..c310be2ec2 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -2,9 +2,11 @@ DVC config objects. """ import os +import errno import configobj from schema import Schema, Optional, And, Use, Regex +from dvc.logger import Logger from dvc.exceptions import DvcException @@ -54,6 +56,9 @@ def is_percent(val): class Config(object): + APPNAME = 'dvc' + APPAUTHOR = 'iterative' + CONFIG = 'config' CONFIG_LOCAL = 'config.local' @@ -63,6 +68,8 @@ class Config(object): SECTION_CORE_REMOTE = 'remote' SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CORE_INTERACTIVE = 'interactive' + SECTION_CORE_ANALYTICS = 'analytics' + SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool)) SECTION_CACHE = 'cache' SECTION_CACHE_DIR = 'dir' @@ -101,6 +108,8 @@ class Config(object): Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)), Optional(SECTION_CORE_INTERACTIVE, default=False): SECTION_CORE_INTERACTIVE_SCHEMA, + Optional(SECTION_CORE_ANALYTICS, + default=True): SECTION_CORE_ANALYTICS_SCHEMA, # backward compatibility Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA, @@ -185,22 +194,74 @@ class Config(object): Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA, } - def __init__(self, dvc_dir): - self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) - self.config_file = os.path.join(dvc_dir, self.CONFIG) - self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) + def __init__(self, dvc_dir=None, validate=True): + self.system_config_file = os.path.join(self.get_system_config_dir(), + self.CONFIG) + self.global_config_file = os.path.join(self.get_global_config_dir(), + self.CONFIG) + + if dvc_dir is not None: + self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) + self.config_file = os.path.join(dvc_dir, self.CONFIG) + self.config_local_file = os.path.join(dvc_dir, self.CONFIG_LOCAL) + else: + self.dvc_dir = None + self.config_file = None + self.config_local_file = None + + self.load(validate=validate) + + @staticmethod + def get_global_config_dir(): + from appdirs import user_config_dir + return user_config_dir(appname=Config.APPNAME, + appauthor=Config.APPAUTHOR) + + @staticmethod + def get_system_config_dir(): + from appdirs import site_config_dir + return site_config_dir(appname=Config.APPNAME, + appauthor=Config.APPAUTHOR) + + @staticmethod + def init(dvc_dir): + config_file = os.path.join(dvc_dir, Config.CONFIG) + open(config_file, 'w+').close() + return Config(dvc_dir) + + def _load(self): + self._system_config = configobj.ConfigObj(self.system_config_file) + self._global_config = configobj.ConfigObj(self.global_config_file) + + if self.config_file is not None: + self._project_config = configobj.ConfigObj(self.config_file) + else: + self._project_config = configobj.ConfigObj() + if self.config_local_file is not None: + self._local_config = configobj.ConfigObj(self.config_local_file) + else: + self._local_config = configobj.ConfigObj() + + self._config = None + + def load(self, validate=True): + self._load() try: - self._config = configobj.ConfigObj(self.config_file) + self._config = configobj.ConfigObj(self.system_config_file) + user = configobj.ConfigObj(self.global_config_file) + config = configobj.ConfigObj(self.config_file) local = configobj.ConfigObj(self.config_local_file) # NOTE: schema doesn't support ConfigObj.Section validation, so we # need to convert our config to dict before passing it to self._config = self._lower(self._config) - local = self._lower(local) - self._config = self._merge(self._config, local) + for c in [user, config, local]: + c = self._lower(c) + self._config = self._merge(self._config, c) - self._config = Schema(self.SCHEMA).validate(self._config) + if validate: + self._config = Schema(self.SCHEMA).validate(self._config) # NOTE: now converting back to ConfigObj self._config = configobj.ConfigObj(self._config, @@ -209,6 +270,75 @@ def __init__(self, dvc_dir): except Exception as ex: raise ConfigError(ex) + def _get_key(self, d, name, add=False): + for k in d.keys(): + if k.lower() == name.lower(): + return k + + if add: + d[name] = {} + return name + + return None + + def save(self, config=None): + if config is not None: + clist = [config] + else: + clist = [self._system_config, + self._global_config, + self._project_config, + self._local_config] + + for conf in clist: + if conf.filename is None: + continue + + try: + Logger.debug("Writing '{}'.".format(conf.filename)) + dname = os.path.dirname(os.path.abspath(conf.filename)) + try: + os.makedirs(dname) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + conf.write() + except Exception as exc: + msg = "Failed to write config '{}'".format(conf.filename) + raise ConfigError(msg, exc) + + def unset(self, config, section, opt=None): + if section not in config.keys(): + raise ConfigError("Section '{}' doesn't exist".format(section)) + + if opt is None: + del config[section] + return + + if opt not in config[section].keys(): + raise ConfigError("Option '{}.{}' doesn't exist".format(section, + opt)) + del config[section][opt] + + if len(config[section]) == 0: + del config[section] + + def set(self, config, section, opt, value): + if section not in config.keys(): + config[section] = {} + + config[section][opt] = value + + def show(self, config, section, opt): + if section not in config.keys(): + raise ConfigError("Section '{}' doesn't exist".format(section)) + + if opt not in config[section].keys(): + raise ConfigError("Option '{}.{}' doesn't exist".format(section, + opt)) + + Logger.info(config[section][opt]) + @staticmethod def _merge(first, second): res = {} @@ -229,9 +359,3 @@ def _lower(config): new_s[key.lower()] = value new_config[s_key.lower()] = new_s return new_config - - @staticmethod - def init(dvc_dir): - config_file = os.path.join(dvc_dir, Config.CONFIG) - open(config_file, 'w+').close() - return Config(dvc_dir) diff --git a/dvc/daemon.py b/dvc/daemon.py index f0d1775d39..a4d715707f 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -42,13 +42,13 @@ def _spawn_posix(self, cmd): close_fds=True, shell=False) - def __call__(self, name): + def __call__(self, args): from dvc.utils import is_binary cmd = [sys.executable] if not is_binary(): cmd += ['-m', 'dvc'] - cmd += ['daemon', name, '-q'] + cmd += ['daemon', '-q'] + args Logger.debug("Trying to spawn '{}'".format(cmd)) diff --git a/dvc/main.py b/dvc/main.py index 1ebc3e6258..1a691f7cc8 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,27 +1,35 @@ from dvc.logger import Logger from dvc.cli import parse_args from dvc.command.base import CmdBase +from dvc.analytics import Analytics +from dvc.cli import DvcParserError +from dvc.project import NotDvcProjectError def main(argv=None): Logger.init() - args = parse_args(argv) + args = None + cmd = None + try: + args = parse_args(argv) - # Init loglevel early in case we'll run - # into errors before setting it properly - CmdBase._set_loglevel(args) + # Init loglevel early in case we'll run + # into errors before setting it properly + CmdBase._set_loglevel(args) - try: cmd = args.func(args) - except Exception as ex: - Logger.error('Initialization error', ex) - return 255 - try: ret = cmd.run_cmd() + except NotDvcProjectError as ex: + Logger.error(str(ex)) + ret = 253 + except DvcParserError: + ret = 254 except Exception as ex: Logger.error('Unexpected error', ex) - return 254 + ret = 255 + + Analytics().send(detach=True, cmd=cmd, args=args, ret=ret) return ret diff --git a/dvc/project.py b/dvc/project.py index 2d965f656d..99749a95aa 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -10,6 +10,12 @@ def __init__(self, msg): super(InitError, self).__init__(msg) +class NotDvcProjectError(DvcException): + def __init__(self, root): + msg = "Not a dvc repository (checked up to mount point {})" + super(NotDvcProjectError, self).__init__(msg.format(root)) + + class ReproductionError(DvcException): def __init__(self, dvc_file_name, ex): self.path = dvc_file_name @@ -20,7 +26,7 @@ def __init__(self, dvc_file_name, ex): class Project(object): DVC_DIR = '.dvc' - def __init__(self, root_dir): + def __init__(self, root_dir=None): from dvc.logger import Logger from dvc.config import Config from dvc.state import State @@ -31,10 +37,13 @@ def __init__(self, root_dir): from dvc.updater import Updater from dvc.prompt import Prompt + root_dir = self._find_root(root_dir) + self.root_dir = os.path.abspath(os.path.realpath(root_dir)) self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR) self.config = Config(self.dvc_dir) + self.scm = SCM(self.root_dir, project=self) self.lock = Lock(self.dvc_dir) # NOTE: storing state and link_state in the repository itself to avoid @@ -55,6 +64,27 @@ def __init__(self, root_dir): self.updater.check() + @staticmethod + def _find_root(root=None): + if root is None: + root = os.getcwd() + else: + root = os.path.abspath(os.path.realpath(root)) + + while True: + dvc_dir = os.path.join(root, Project.DVC_DIR) + if os.path.isdir(dvc_dir): + return root + if os.path.ismount(root): + break + root = os.path.dirname(root) + raise NotDvcProjectError(root) + + @staticmethod + def _find_dvc_dir(root=None): + root_dir = Project._find_root(root) + return os.path.join(root_dir, Project.DVC_DIR) + def _remind_to_git_add(self): if len(self._files_to_git_add) == 0: return diff --git a/dvc/updater.py b/dvc/updater.py index 5e5cd3505a..2e8cff96fc 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -68,8 +68,7 @@ def fetch(self, detach=True): from dvc.daemon import Daemon if detach: - d = Daemon() - d('updater') + Daemon()(['updater']) return self._with_lock(self._get_latest_version, 'fetching') diff --git a/requirements.txt b/requirements.txt index a0922bbe07..6d09054065 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ grandalf==0.6 pydot>=1.2.4 asciimatics>=1.10.0 distro>=1.3.0 +appdirs>=1.4.3 diff --git a/setup.py b/setup.py index 708f83b5ea..a4fe909925 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ "grandalf==0.6", "asciimatics>=1.10.0", "distro>=1.3.0", + "appdirs>=1.4.3", ] # Extra dependencies for remote integrations diff --git a/tests/test_analytics.py b/tests/test_analytics.py new file mode 100644 index 0000000000..4ff9a94cd7 --- /dev/null +++ b/tests/test_analytics.py @@ -0,0 +1,62 @@ +import os +import mock +import requests +import tempfile + +from dvc.main import main +from dvc.analytics import Analytics +from tests.basic_env import TestDvc, TestGit, TestDir + + +def _clean_getenv(key, default=None): + """ + Remove env vars that affect dvc behavior in tests + """ + if key in ['DVC_TEST', 'CI']: + return None + return os.environ.get(key, default) + + +class TestAnalytics(TestDir): + def test(self): + a = Analytics() + a._collect() + self.assertTrue(isinstance(a.info, dict)) + self.assertNotEqual(a.info, {}) + self.assertTrue(a.PARAM_USER_ID in a.info.keys()) + self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys()) + self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {}) + + @mock.patch.object(os, 'getenv', new=_clean_getenv) + @mock.patch('requests.post') + def test_send(self, mockpost): + ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + self.assertEqual(ret, 0) + + self.assertTrue(mockpost.called) + + @mock.patch.object(os, 'getenv', new=_clean_getenv) + @mock.patch.object(requests, 'post', + side_effect=requests.exceptions.RequestException()) + def test_send_failed(self, mockpost): + ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + self.assertEqual(ret, 0) + + self.assertTrue(mockpost.called) + + +class TestAnalyticsGit(TestAnalytics, TestGit): + pass + + +class TestAnalyticsDvc(TestAnalytics, TestDvc): + @mock.patch('requests.post') + def test_send_disabled(self, mockpost): + ret = main(['config', 'core.analytics', 'false']) + self.assertEqual(ret, 0) + + with mock.patch.object(os, 'getenv', new=_clean_getenv): + ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + self.assertEqual(ret, 0) + + self.assertFalse(mockpost.called) diff --git a/tests/test_config.py b/tests/test_config.py index 93e8d5c9b8..78be9edcaa 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -89,6 +89,8 @@ def test_non_existing(self): def test_failed_write(self): class A(object): + system = False + glob = False local = False name = 'core.remote' value = 'myremote' @@ -98,5 +100,5 @@ class A(object): cmd = CmdConfig(args) cmd.configobj.write = None - ret = cmd.save() + ret = cmd.run() self.assertNotEqual(ret, 0) diff --git a/tests/test_remote.py b/tests/test_remote.py index 9f1f575bcf..2e1d57ccdf 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -30,6 +30,8 @@ def test(self): def test_failed_write(self): class A(object): + system = False + glob = False local = False name = 'myremote' url = 's3://remote'