XDG support

- backport MycroftAI#2794
- optional XDG (configurable, ensure backwards compat)
- dont auto create xdg paths (this was triggering some race conditions that killed the whole process when multiple places attempted to create the folder at same time, a simple string definition should not be creating directories)
- includes https://github.com/HelloChatterbox/HolmesV/pull/22 which was dropped, it merged the wrong (incomplete, skills only) XDG PR, this was originally merged before the mock-msm PR, apologies for the out of order commit history

mycroft/api/__init__.py

@@ -20,8 +20,6 @@
 from requests import HTTPError, RequestException
 
 from mycroft.configuration import Configuration
-from mycroft.configuration.config import DEFAULT_CONFIG, SYSTEM_CONFIG, \
-    USER_CONFIG, LocalConf
 from mycroft.identity import IdentityManager, identity_lock
 from mycroft.version import VersionManager
 from mycroft.util import get_arch, connected, LOG
@@ -49,12 +47,9 @@ class Api:
     def __init__(self, path):
         self.path = path
 
-        # Load the config, skipping the REMOTE_CONFIG since we are
+        # Load the config, skipping the remote config since we are
         # getting the info needed to get to it!
-        config = Configuration.get([DEFAULT_CONFIG,
-                                    SYSTEM_CONFIG,
-                                    USER_CONFIG],
-                                   cache=False)
+        config = Configuration.get(cache=False, remote=False)
         config_server = config.get("server")
         self.url = config_server.get("url")
         self.version = config_server.get("version")
@@ -243,9 +238,7 @@ def activate(self, state, token):
         platform_build = ""
 
         # load just the local configs to get platform info
-        config = Configuration.get([SYSTEM_CONFIG,
-                                    USER_CONFIG],
-                                   cache=False)
+        config = Configuration.get(cache=False, remote=False)
         if "enclosure" in config:
             platform = config.get("enclosure").get("platform", "unknown")
             platform_build = config.get("enclosure").get("platform_build", "")
@@ -267,9 +260,7 @@ def update_version(self):
         platform_build = ""
 
         # load just the local configs to get platform info
-        config = Configuration.get([SYSTEM_CONFIG,
-                                    USER_CONFIG],
-                                   cache=False)
+        config = Configuration.get(cache=False, remote=False)
         if "enclosure" in config:
             platform = config.get("enclosure").get("platform", "unknown")
             platform_build = config.get("enclosure").get("platform_build", "")
@@ -551,8 +542,5 @@ def check_remote_pairing(ignore_errors):
 
 
 def is_backend_disabled():
-    configs = [LocalConf(DEFAULT_CONFIG),
-               LocalConf(SYSTEM_CONFIG),
-               LocalConf(USER_CONFIG)]
-    config = Configuration.load_config_stack(configs)
+    config = Configuration.get(cache=False, remote=False)
     return config["server"].get("disabled") or False

mycroft/client/enclosure/__main__.py

@@ -17,7 +17,7 @@
 This provides any "enclosure" specific functionality, for example GUI or
 control over the Mark-1 Faceplate.
 """
-from mycroft.configuration import LocalConf, SYSTEM_CONFIG
+from mycroft.configuration import Configuration
 from mycroft.util.log import LOG
 from mycroft.util import wait_for_exit_signal, reset_sigint_handler
 
@@ -72,8 +72,8 @@ def main(ready_hook=on_ready, error_hook=on_error, stopping_hook=on_stopping):
     only the GUI bus will be started.
     """
     # Read the system configuration
-    system_config = LocalConf(SYSTEM_CONFIG)
-    platform = system_config.get("enclosure", {}).get("platform")
+    config = Configuration.get(remote=False)
+    platform = config.get("enclosure", {}).get("platform")
 
     enclosure = create_enclosure(platform)
     if enclosure:

mycroft/client/enclosure/mark1/__init__.py

@@ -14,7 +14,6 @@
 #
 import subprocess
 import time
-import sys
 from alsaaudio import Mixer
 from threading import Thread, Timer
 
@@ -163,6 +162,9 @@ def process(self, data):
         if "unit.factory-reset" in data:
             self.bus.emit(Message("speak", {
                 'utterance': mycroft.dialog.get("reset to factory defaults")}))
+            subprocess.call(
+                'rm ~/.config/mycroft/identity/identity2.json',
+                shell=True)
             subprocess.call(
                 'rm ~/.mycroft/identity/identity2.json',
                 shell=True)

mycroft/client/speech/hotword_factory.py

@@ -26,13 +26,15 @@
 from threading import Timer, Thread
 from time import time, sleep
 from urllib.error import HTTPError
+from xdg import BaseDirectory as XDG
 import speech_recognition as sr
 
 import requests
 
-from mycroft.configuration import Configuration, LocalConf, USER_CONFIG
-from mycroft.util.monotonic_event import MonotonicEvent
+from mycroft.configuration import Configuration, LocalConf
+from mycroft.configuration.locations import OLD_USER_CONFIG
 from mycroft.util.log import LOG
+from mycroft.util.monotonic_event import MonotonicEvent
 from mycroft.util.plugins import load_plugin
 
 RECOGNIZER_DIR = join(abspath(dirname(__file__)), "recognizer")
@@ -197,7 +199,32 @@ def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"):
         from precise_runner import (
             PreciseRunner, PreciseEngine, ReadWriteStream
         )
-        local_conf = LocalConf(USER_CONFIG)
+
+        conf = Configuration.get(remote=False)
+        if conf.get("disable_xdg"):
+            local_conf = {}
+        else:
+            # We need to save to a writeable location, but the key we need
+            # might be stored in a different, unwriteable, location
+            # Make sure we pick the key we need from wherever it's located,
+            # but save to a writeable location only
+
+            local_conf = LocalConf(join(XDG.xdg_config_home, 'mycroft',
+                                        'mycroft.conf'))
+
+            for path in XDG.load_config_paths('mycroft'):
+                conf = LocalConf(join(path, 'mycroft.conf'))
+                # If the current config contains the precise key use it,
+                # otherwise continue to the next file
+                if conf.get('precise', None) is not None:
+                    local_conf['precise'] = conf.get('precise', None)
+                    break
+
+        # If the key is not found yet, it might still exist on the old
+        # (deprecated) location
+        if local_conf.get('precise', None) is None:
+            local_conf = LocalConf(OLD_USER_CONFIG)
+
         if (local_conf.get('precise', {}).get('dist_url') ==
                 'http://bootstrap.mycroft.ai/artifacts/static/daily/'):
             del local_conf['precise']['dist_url']
@@ -255,7 +282,10 @@ def update_precise(self, precise_config):
 
     @property
     def folder(self):
-        return join(expanduser('~'), '.mycroft', 'precise')
+        old_path = join(expanduser('~'), '.mycroft', 'precise')
+        if os.path.isdir(old_path):
+            return old_path
+        return join(XDG.xdg_data_home, 'mycroft', "precise")
 
     @property
     def install_destination(self):

mycroft/client/text/text_client.py

@@ -15,6 +15,7 @@
 import sys
 import io
 from math import ceil
+from xdg import BaseDirectory as XDG
 
 from mycroft.client.text.gui_server import start_qml_gui
 
@@ -142,7 +143,7 @@ def handleNonAscii(text):
 ##############################################################################
 # Settings
 
-config_file = os.path.join(os.path.expanduser("~"), ".mycroft_cli.conf")
+filename = "mycroft_cli.conf"
 
 
 def load_mycroft_config(bus):
@@ -171,6 +172,38 @@ def load_settings():
     global max_log_lines
     global show_meter
 
+    config_file = None
+
+    # Old location
+    path = os.path.join(os.path.expanduser("~"), ".mycroft_cli.conf")
+
+    core_conf = Configuration.get(remote=False)
+    if core_conf.get("disable_xdg"):
+        config_file = path
+    elif os.path.isfile(path):
+        LOG.warning(" ===============================================")
+        LOG.warning(" ==             DEPRECATION WARNING           ==")
+        LOG.warning(" ===============================================")
+        LOG.warning(" You still have a config file at " +
+                    path)
+        LOG.warning(" Note that this location is deprecated and will" +
+                    " not be used in the future")
+        LOG.warning(" Please move it to " +
+                    os.path.join(XDG.xdg_config_home, 'mycroft', filename))
+        config_file = path
+
+    # Check XDG_CONFIG_DIR
+    if config_file is None:
+        for p in XDG.load_config_paths('mycroft'):
+            file = os.path.join(p, filename)
+            if os.path.isfile(file):
+                config_file = file
+                break
+
+    # Check /etc/mycroft
+    if config_file is None:
+        config_file = os.path.join("/etc/mycroft", filename)
+
     try:
         with io.open(config_file, 'r') as f:
             config = json.load(f)
@@ -196,6 +229,17 @@ def save_settings():
     config["show_last_key"] = show_last_key
     config["max_log_lines"] = max_log_lines
     config["show_meter"] = show_meter
+
+
+    # Old location
+    path = os.path.join(os.path.expanduser("~"), ".mycroft_cli.conf")
+
+    core_conf = Configuration.get(remote=False)
+    if core_conf.get("disable_xdg"):
+        config_file = path
+    else:
+        config_file = os.path.join(XDG.xdg_config_home, 'mycroft', filename)
+
     with io.open(config_file, 'w') as f:
         f.write(str(json.dumps(config, ensure_ascii=False)))
 

mycroft/configuration/config.py

@@ -17,13 +17,16 @@
 import re
 import json
 import inflection
-from os.path import exists, isfile
+from os.path import exists, isfile, join
 from requests import RequestException
+from xdg import BaseDirectory as XDG
+
 from mycroft.util.json_helper import load_commented_json, merge_dict
 from mycroft.util.log import LOG
 
 from mycroft.configuration.locations import (DEFAULT_CONFIG, SYSTEM_CONFIG,
-                                             USER_CONFIG, WEB_CONFIG_CACHE)
+                                             OLD_USER_CONFIG, WEB_CONFIG_CACHE,
+                                             USER_CONFIG)
 
 
 def is_remote_list(values):
@@ -82,6 +85,7 @@ def translate_list(config, values):
 
 class LocalConf(dict):
     """Config dictionary from file."""
+
     def __init__(self, path):
         super().__init__()
         if path:
@@ -123,18 +127,22 @@ def merge(self, conf):
 
 class RemoteConf(LocalConf):
     """Config dictionary fetched from mycroft.ai."""
+
     def __init__(self, cache=None):
-        super().__init__(None)
+        super(RemoteConf, self).__init__(None)
+
         cache = cache or WEB_CONFIG_CACHE
-        from mycroft.api import is_paired
-        if not is_paired():
-            self.load_local(cache)
-            return
+
         try:
             # Here to avoid cyclic import
+            from mycroft.api import is_paired
             from mycroft.api import DeviceApi
             from mycroft.api import is_backend_disabled
 
+            if not is_paired():
+                self.load_local(cache)
+                return
+
             if is_backend_disabled():
                 # disable options that require backend
                 config = {
@@ -186,7 +194,7 @@ class Configuration:
     __patch = {}  # Patch config that skills can update to override config
 
     @staticmethod
-    def get(configs=None, cache=True):
+    def get(configs=None, cache=True, remote=True):
         """Get configuration
 
         Returns cached instance if available otherwise builds a new
@@ -195,30 +203,83 @@ def get(configs=None, cache=True):
         Arguments:
             configs (list): List of configuration dicts
             cache (boolean): True if the result should be cached
+            remote (boolean): False if the Mycroft Home settings shouldn't
+                              be loaded
+
 
         Returns:
             (dict) configuration dictionary.
         """
         if Configuration.__config:
             return Configuration.__config
         else:
-            return Configuration.load_config_stack(configs, cache)
+            return Configuration.load_config_stack(configs, cache, remote)
 
     @staticmethod
-    def load_config_stack(configs=None, cache=False):
+    def load_config_stack(configs=None, cache=False, remote=True):
         """Load a stack of config dicts into a single dict
 
         Arguments:
             configs (list): list of dicts to load
             cache (boolean): True if result should be cached
-
+            remote (boolean): False if the Mycroft Home settings shouldn't
+                              be loaded
         Returns:
             (dict) merged dict of all configuration files
         """
+
         if not configs:
-            configs = [LocalConf(DEFAULT_CONFIG), RemoteConf(),
-                       LocalConf(SYSTEM_CONFIG), LocalConf(USER_CONFIG),
-                       Configuration.__patch]
+            configs = configs or []
+
+            # first let's load all non XDG configs, and check if XDG is disabled
+            # TODO deprecate this once mycroft-core supports XDG
+            _config = Configuration.load_config_stack(
+                [DEFAULT_CONFIG, SYSTEM_CONFIG, OLD_USER_CONFIG],
+                cache=False, remote=False)
+
+            # First use the patched config
+            configs.append(Configuration.__patch)
+
+            if _config.get("disable_xdg"):
+                # Then check the old user config
+                configs.append(LocalConf(OLD_USER_CONFIG))
+            else:
+                # Then use XDG config
+                # This includes both the user config and
+                # /etc/xdg/mycroft/mycroft.conf
+                for p in XDG.load_config_paths('mycroft'):
+                    configs.append(LocalConf(join(p, 'mycroft.conf')))
+
+                # Then check the old user config
+                if isfile(OLD_USER_CONFIG):
+                    LOG.warning(" ===============================================")
+                    LOG.warning(" ==             DEPRECATION WARNING           ==")
+                    LOG.warning(" ===============================================")
+                    LOG.warning(" You still have a config file at " +
+                                OLD_USER_CONFIG)
+                    LOG.warning(" Note that this location is deprecated and will" +
+                                " not be used in the future")
+                    LOG.warning(" Please move it to " +
+                                join(XDG.xdg_config_home, 'mycroft'))
+                    configs.append(LocalConf(OLD_USER_CONFIG))
+
+                # Then check the XDG user config
+                if isfile(USER_CONFIG):
+                    configs.append(LocalConf(USER_CONFIG))
+
+            # Then use remote config
+            if remote:
+                configs.append(RemoteConf())
+
+            # Then use the system config (/etc/mycroft/mycroft.conf)
+            configs.append(LocalConf(SYSTEM_CONFIG))
+
+            # Then use the config that comes with the package
+            configs.append(LocalConf(DEFAULT_CONFIG))
+
+            # Make sure we reverse the array, as merge_dict will put every new
+            # file on top of the previous one
+            configs = reversed(configs)
         else:
             # Handle strings in stack
             for index, item in enumerate(configs):