Skill location utilities

ovos_utils/skills/__init__.py

@@ -1,7 +1,7 @@
-from ovos_utils.configuration import read_mycroft_config, update_mycroft_config, get_xdg_data_save_path
+from ovos_config.config import read_mycroft_config, update_mycroft_config
 from ovos_utils.messagebus import wait_for_reply
-from os.path import join, isdir, isfile
-from os import listdir
+from ovos_utils.skills.locations import get_default_skills_directory, get_installed_skill_ids
+from ovos_utils.log import LOG
 
 
 def get_non_properties(obj):
@@ -87,43 +87,12 @@ def make_priority_skill(skill, config=None):
 
 
 def get_skills_folder(config=None):
-    # once XDG PR is merged skills folder will no longer be configurable,
-    # skills are moved automatically to new locations
-    # this is already live in mycroft-lib
-    xdg_skills = join(get_xdg_data_save_path(), 'skills')
-    if isdir(xdg_skills):
-        return xdg_skills
-
-    # read user defined location
-    config = config or read_mycroft_config()
-    if config:
-        skill_folder = config["skills"].get("msm", {}).get("directory")
-        if skill_folder:
-            return join(config["data_dir"], skill_folder)
-
-    # check if default path exists
-    elif isdir("/opt/mycroft/skills"):
-        return "/opt/mycroft/skills"
-
-    # .conf not found, xdg directory not detected, default path not
-    # detected, doesn't look like we are running mycroft-core
-    return None
+    LOG.warning("This reference is deprecated, use "
+                "`ovos_utils.skills.locations.get_default_skill_dir")
+    return get_default_skills_directory(config)
 
 
 def get_installed_skills(config=None):
-    skills_dir = get_skills_folder(config)
-    installed_skills = []
-    if skills_dir:
-        for skill_id in listdir(skills_dir):
-            skill_path = join(skills_dir, skill_id)
-            if not isdir(skill_path):
-                continue
-            skill_file = join(skill_path, "__init__.py")
-            if not isfile(skill_file):
-                continue
-            with open(skill_file) as f:
-                if "def create_skill(" not in f.read():
-                    continue
-            installed_skills.append(skill_id)
-
-    return installed_skills
+    LOG.warning("This reference is deprecated, use "
+                "`ovos_utils.skills.locations.get_installed_skill_ids")
+    return get_installed_skill_ids(config)

ovos_utils/skills/locations.py

@@ -0,0 +1,148 @@
+from os.path import join, isdir, dirname, expanduser, isfile
+from os import makedirs, listdir
+from typing import List, Optional
+from ovos_config.locations import get_xdg_data_save_path, get_xdg_data_dirs
+from ovos_config.config import Configuration
+from ovos_utils.log import LOG
+
+
+def get_installed_skill_ids(conf: Optional[dict] = None) -> List[str]:
+    """
+    Gets a list of `skill_id`s for all installed skills
+    Args:
+        conf: Configuration, else loads from ovos_config.config.Configuration
+    Returns:
+        list of `skill_id` strings for all installed skills
+    """
+    _, skill_ids = get_plugin_skills()
+    for d in get_skill_directories(conf):
+        for skill_dir in listdir(d):
+            if isdir(join(d, skill_dir)) and isfile(join(d, skill_dir,
+                                                         "__init__.py")):
+                if skill_dir in skill_ids:
+                    LOG.info(f"{skill_dir} installed as plugin and local dir")
+                    continue
+                skill_ids.append(skill_dir)
+    return skill_ids
+
+
+def get_skill_directories(conf: Optional[dict] = None) -> List[str]:
+    """ returns list of skill directories ordered by expected loading order
+    This corresponds to:
+    - XDG_DATA_DIRS
+    - default directory (see get_default_skills_directory method for details)
+    - user defined extra directories
+    Each directory contains individual skill folders to be loaded
+    If a skill exists in more than one directory (same folder name) previous instances will be ignored
+        ie. directories at the end of the list have priority over earlier directories
+    NOTE: empty folders are interpreted as disabled skills
+    new directories can be defined in mycroft.conf by specifying a full path
+    each extra directory is expected to contain individual skill folders to be loaded
+    the xdg folder name can also be changed, it defaults to "skills"
+        eg. ~/.local/share/mycroft/FOLDER_NAME
+    {
+        "skills": {
+            "directory": "skills",
+            "extra_directories": ["path/to/extra/dir/to/scan/for/skills"]
+        }
+    }
+    Args:
+        conf: Configuration, else loads from ovos_config.config.Configuration
+    Returns:
+        list of fully-qualified directories containing non-plugin skills
+    """
+    # the contents of each skills directory must be individual skill folders
+    # we are still dependent on the mycroft-core structure of skill_id/__init__.py
+
+    conf = conf or Configuration()
+
+    # load all valid XDG paths
+    # NOTE: skills are actually code, but treated as user data!
+    # they should be considered applets rather than full applications
+    skill_locations = list(reversed(
+        [join(p, "skills") for p in get_xdg_data_dirs() if
+         isdir(join(p, "skills"))]
+    ))
+
+    # load the default skills folder
+    # only meaningful if xdg support is disabled
+    default = get_default_skills_directory(conf)
+    if default not in skill_locations:
+        skill_locations.append(default)
+
+    # load additional explicitly configured directories
+    conf = conf.get("skills") or {}
+    # extra_directories is a list of directories containing skill subdirectories
+    # NOT a list of individual skill folders
+    # preserve order while removing any duplicate entries
+    extra_dirs = (expanduser(d) for d in conf.get("extra_directories") or [])
+    for d in extra_dirs:
+        if isdir(d) and d not in skill_locations:
+            skill_locations.append(d)
+    return skill_locations
+
+
+def get_default_skills_directory(conf: Optional[dict] = None) -> str:
+    """ return default directory to scan for skills
+    This is only meaningful if xdg is disabled in ovos.conf
+    If xdg is enabled then data_dir is always XDG_DATA_DIR
+    If xdg is disabled then data_dir by default corresponds to /opt/mycroft
+    users can define the data directory in mycroft.conf
+    the skills folder name (relative to data_dir) can also be defined there
+    NOTE: folder name also impacts all XDG skill directories!
+    {
+        "data_dir": "/opt/mycroft",
+        "skills": {
+            "directory": "skills"
+        }
+    }
+    Args:
+        conf: Configuration, else loads from ovos_config.config.Configuration
+    Returns:
+        Absolute path to default skills directory
+    """
+    conf = conf or Configuration()
+    path_override = conf["skills"].get("directory_override")
+
+    # if .conf wants to use a specific path, use it!
+    if path_override:
+        LOG.warning("'directory_override' is deprecated!\n"
+                    "It will no longer be supported after version 0.0.3\n"
+                    "add the new path to 'extra_directories' instead")
+        skills_folder = expanduser(path_override)
+    elif conf["skills"].get("extra_directories") and \
+            len(conf["skills"].get("extra_directories")) > 0:
+        skills_folder = expanduser(conf["skills"]["extra_directories"][0])
+    else:
+        skills_folder = join(get_xdg_data_save_path(), "skills")
+    # create folder if needed
+    try:
+        makedirs(skills_folder, exist_ok=True)
+    except PermissionError:  # old style /opt/mycroft/skills not available
+        skills_folder = join(get_xdg_data_save_path(), "skills")
+        makedirs(skills_folder, exist_ok=True)
+
+    return skills_folder
+
+
+def get_plugin_skills() -> (list, list):
+    """
+    Get the package directories for any pip installed skill plugins
+    Returns:
+        lists of skill directories and plugin skill IDs
+    """
+    import importlib.util
+    try:
+        from ovos_plugin_manager.skills import find_skill_plugins
+    except ImportError:
+        LOG.warning("ovos-plugin-manager not available to load plugin skills")
+        return [], []
+    skill_dirs = list()
+    plugins = find_skill_plugins()
+    skill_ids = list(plugins.keys())
+    for skill_class in plugins.values():
+        skill_dir = dirname(importlib.util.find_spec(
+            skill_class.__module__).origin)
+        skill_dirs.append(skill_dir)
+    LOG.info(f"Located plugin skills: {skill_ids}")
+    return skill_dirs, skill_ids

test/unittests/test_skills.py

@@ -0,0 +1,79 @@
+import unittest
+from os import environ
+from os.path import isdir, join, dirname
+from unittest.mock import patch
+
+
+class TestLocations(unittest.TestCase):
+    @patch("ovos_utils.skills.locations.get_plugin_skills")
+    def test_get_installed_skill_ids(self, plugins):
+        plugins.return_value = (['plugin_dir', 'plugin_dir_2'],
+                                ['plugin_id', 'plugin_id_2'])
+        from ovos_utils.skills.locations import get_installed_skill_ids
+        environ["XDG_DATA_DIRS"] = join(dirname(__file__), "test_skills_xdg")
+        config = {"skills": {
+            "extra_directories": [join(dirname(__file__), "test_skills_dir")]
+        }}
+        skill_ids = get_installed_skill_ids(config)
+        self.assertEqual(set(skill_ids), {"plugin_id", "plugin_id_2",
+                                          "skill-test-1.openvoiceos",
+                                          "skill-test-2.openvoiceos"})
+
+    def test_get_skill_directories(self):
+        from ovos_utils.skills.locations import get_skill_directories
+
+        # Default behavior, only one valid XDG path
+        environ["XDG_DATA_DIRS"] = environ["XDG_DATA_HOME"] = \
+            join(dirname(__file__), "test_skills_xdg")
+        config = {"skills": {"extra_directories": []}}
+        default_dir = join(dirname(__file__), "test_skills_xdg",
+                           "mycroft", "skills")
+        self.assertEqual(get_skill_directories(config), [default_dir])
+
+        # Define single extra directory to append
+        extra_dir = join(dirname(__file__), "test_skills_dir")
+        config['skills']['extra_directories'] = [extra_dir]
+        self.assertEqual(get_skill_directories(config),
+                         [default_dir, extra_dir])
+
+        # Define duplicated directories in extra_directories
+        config['skills']['extra_directories'] += [extra_dir, default_dir]
+        self.assertEqual(get_skill_directories(config),
+                         [default_dir, extra_dir])
+
+        # Define invalid directories in extra_directories
+        config['skills']['extra_directories'] += ["/not/a/directory"]
+        self.assertEqual(get_skill_directories(config),
+                         [default_dir, extra_dir])
+
+    def test_get_default_skills_directory(self):
+        from ovos_utils.skills.locations import get_default_skills_directory
+        test_skills_dir = join(dirname(__file__), "test_skills_dir")
+
+        # Configured override (legacy)
+        config = {"skills": {"directory_override": test_skills_dir}}
+        self.assertEqual(get_default_skills_directory(config), test_skills_dir)
+
+        # Configured extra_directories
+        config = {"skills": {"extra_directories": [test_skills_dir, "/tmp"]}}
+        self.assertEqual(get_default_skills_directory(config), test_skills_dir)
+
+        environ["XDG_DATA_HOME"] = join(dirname(__file__), "test_skills_xdg")
+        xdg_skills_dir = join(dirname(__file__), "test_skills_xdg",
+                              "mycroft", "skills")
+        # XDG (undefined extra_directories)
+        config = {"skills": {}}
+        self.assertEqual(get_default_skills_directory(config), xdg_skills_dir)
+
+        # XDG (empty extra_directories)
+        config = {"skills": {"extra_directories": []}}
+        self.assertEqual(get_default_skills_directory(config), xdg_skills_dir)
+
+    def test_get_plugin_skills(self):
+        from ovos_utils.skills.locations import get_plugin_skills
+        dirs, ids = get_plugin_skills()
+        for d in dirs:
+            self.assertTrue(isdir(d))
+        for s in ids:
+            self.assertIsInstance(s, str)
+        self.assertEqual(len(dirs), len(ids))