Implement external custom components installing from YAML

.github/workflows/ci.yml

@@ -115,7 +115,7 @@ jobs:
         uses: actions/cache@v1
         with:
           path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
           restore-keys: |
             test-home-platformio-${{ matrix.test }}-
       - name: Set up environment

.github/workflows/release-dev.yml

@@ -112,7 +112,7 @@ jobs:
         uses: actions/cache@v1
         with:
           path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
           restore-keys: |
             test-home-platformio-${{ matrix.test }}-
       - name: Set up environment

.github/workflows/release.yml

@@ -111,7 +111,7 @@ jobs:
         uses: actions/cache@v1
         with:
           path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
           restore-keys: |
             test-home-platformio-${{ matrix.test }}-
       - name: Set up environment

esphome/__main__.py

@@ -638,10 +638,10 @@ def run_esphome(argv):
         _LOGGER.error("Missing configuration parameter, see esphome --help.")
         return 1
 
-    if sys.version_info < (3, 6, 0):
+    if sys.version_info < (3, 7, 0):
         _LOGGER.error(
-            "You're running ESPHome with Python <3.6. ESPHome is no longer compatible "
-            "with this Python version. Please reinstall ESPHome with Python 3.6+"
+            "You're running ESPHome with Python <3.7. ESPHome is no longer compatible "
+            "with this Python version. Please reinstall ESPHome with Python 3.7+"
         )
         return 1
 

esphome/components/external_components/__init__.py

@@ -0,0 +1,203 @@
+import re
+import logging
+from pathlib import Path
+import subprocess
+import hashlib
+import datetime
+
+import esphome.config_validation as cv
+from esphome.const import (
+    CONF_COMPONENTS,
+    CONF_SOURCE,
+    CONF_URL,
+    CONF_TYPE,
+    CONF_EXTERNAL_COMPONENTS,
+    CONF_PATH,
+)
+from esphome.core import CORE
+from esphome import loader
+
+_LOGGER = logging.getLogger(__name__)
+
+DOMAIN = CONF_EXTERNAL_COMPONENTS
+
+TYPE_GIT = "git"
+TYPE_LOCAL = "local"
+CONF_REFRESH = "refresh"
+CONF_REF = "ref"
+
+
+def validate_git_ref(value):
+    if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None:
+        raise cv.Invalid("Not a valid git ref")
+    return value
+
+
+GIT_SCHEMA = {
+    cv.Required(CONF_URL): cv.url,
+    cv.Optional(CONF_REF): validate_git_ref,
+}
+LOCAL_SCHEMA = {
+    cv.Required(CONF_PATH): cv.directory,
+}
+
+
+def validate_source_shorthand(value):
+    if not isinstance(value, str):
+        raise cv.Invalid("Shorthand only for strings")
+    try:
+        return SOURCE_SCHEMA({CONF_TYPE: TYPE_LOCAL, CONF_PATH: value})
+    except cv.Invalid:
+        pass
+    # Regex for GitHub repo name with optional branch/tag
+    # Note: git allows other branch/tag names as well, but never seen them used before
+    m = re.match(
+        r"github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)(?:@([a-zA-Z0-9\-_.\./]+))?",
+        value,
+    )
+    if m is None:
+        raise cv.Invalid(
+            "Source is not a file system path or in expected github://username/name[@branch-or-tag] format!"
+        )
+    conf = {
+        CONF_TYPE: TYPE_GIT,
+        CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git",
+    }
+    if m.group(3):
+        conf[CONF_REF] = m.group(3)
+    return SOURCE_SCHEMA(conf)
+
+
+def validate_refresh(value: str):
+    if value.lower() == "always":
+        return validate_refresh("0s")
+    if value.lower() == "never":
+        return validate_refresh("1000y")
+    return cv.positive_time_period_seconds(value)
+
+
+SOURCE_SCHEMA = cv.Any(
+    validate_source_shorthand,
+    cv.typed_schema(
+        {
+            TYPE_GIT: cv.Schema(GIT_SCHEMA),
+            TYPE_LOCAL: cv.Schema(LOCAL_SCHEMA),
+        }
+    ),
+)
+
+
+CONFIG_SCHEMA = cv.ensure_list(
+    {
+        cv.Required(CONF_SOURCE): SOURCE_SCHEMA,
+        cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, validate_refresh),
+        cv.Optional(CONF_COMPONENTS, default="all"): cv.Any(
+            "all", cv.ensure_list(cv.string)
+        ),
+    }
+)
+
+
+def to_code(config):
+    pass
+
+
+def _compute_destination_path(key: str) -> Path:
+    base_dir = Path(CORE.config_dir) / ".esphome" / DOMAIN
+    h = hashlib.new("sha256")
+    h.update(key.encode())
+    return base_dir / h.hexdigest()[:8]
+
+
+def _handle_git_response(ret):
+    if ret.stderr:
+        err_str = ret.stderr.decode("utf-8")
+        lines = [x.strip() for x in err_str.splitlines()]
+        if lines[-1].startswith("fatal:"):
+            raise cv.Invalid(lines[-1][len("fatal: ") :])
+        raise cv.Invalid(err_str)
+
+
+def _process_single_config(config: dict):
+    conf = config[CONF_SOURCE]
+    if conf[CONF_TYPE] == TYPE_GIT:
+        key = f"{conf[CONF_URL]}@{conf.get(CONF_REF)}"
+        repo_dir = _compute_destination_path(key)
+        if not repo_dir.is_dir():
+            cmd = ["git", "clone", "--depth=1"]
+            if CONF_REF in conf:
+                cmd += ["--branch", conf[CONF_REF]]
+            cmd += [conf[CONF_URL], str(repo_dir)]
+            ret = subprocess.run(cmd, capture_output=True, check=False)
+            _handle_git_response(ret)
+
+        else:
+            # Check refresh needed
+            file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD")
+            # On first clone, FETCH_HEAD does not exists
+            if not file_timestamp.exists():
+                file_timestamp = Path(repo_dir / ".git" / "HEAD")
+            age = datetime.datetime.now() - datetime.datetime.fromtimestamp(
+                file_timestamp.stat().st_mtime
+            )
+            if age.seconds > config[CONF_REFRESH].total_seconds:
+                _LOGGER.info("Executing git pull %s", key)
+                cmd = ["git", "pull"]
+                ret = subprocess.run(
+                    cmd, cwd=repo_dir, capture_output=True, check=False
+                )
+                _handle_git_response(ret)
+
+        dest_dir = repo_dir
+    elif conf[CONF_TYPE] == TYPE_LOCAL:
+        dest_dir = Path(conf[CONF_PATH])
+    else:
+        raise NotImplementedError()
+
+    try:
+        cv.directory(dest_dir / "esphome" / "components")
+        components_dir = dest_dir / "esphome" / "components"
+    except cv.Invalid as err:
+        try:
+            cv.directory(dest_dir / "components")
+            components_dir = dest_dir / "components"
+        except cv.Invalid:
+            raise cv.Invalid(
+                "Could not find components folder for source. Please check the source contains a 'components' or 'esphome/components' folder",
+                [CONF_SOURCE],
+            ) from err
+
+    if config[CONF_COMPONENTS] == "all":
+        num_components = len(list(components_dir.glob("*/__init__.py")))
+        if num_components > 100:
+            # Prevent accidentally including all components from an esphome fork/branch
+            # In this case force the user to manually specify which components they want to include
+            raise cv.Invalid(
+                "This source is an ESPHome fork or branch. Please manually specify the components you want to import using the 'components' key",
+                [CONF_COMPONENTS],
+            )
+        allowed_components = None
+    else:
+        for i, name in enumerate(config[CONF_COMPONENTS]):
+            expected = components_dir / name / "__init__.py"
+            try:
+                cv.file_(expected)
+            except cv.Invalid as err:
+                raise cv.Invalid(
+                    f"Could not find __init__.py file for component {name}. Please check the component is defined by this source (search path: {expected})",
+                    [CONF_COMPONENTS, i],
+                ) from err
+        allowed_components = config[CONF_COMPONENTS]
+
+    loader.install_meta_finder(components_dir, allowed_components=allowed_components)
+
+
+def do_external_components_pass(config: dict) -> None:
+    conf = config.get(DOMAIN)
+    if conf is None:
+        return
+    with cv.prepend_path(DOMAIN):
+        conf = CONFIG_SCHEMA(conf)
+        for i, c in enumerate(conf):
+            with cv.prepend_path(i):
+                _process_single_config(c)

esphome/components/http_request/__init__.py

@@ -13,7 +13,7 @@
     CONF_URL,
 )
 from esphome.core import CORE, Lambda
-from esphome.core_config import PLATFORMIO_ESP8266_LUT
+from esphome.core.config import PLATFORMIO_ESP8266_LUT
 
 DEPENDENCIES = ["network"]
 AUTO_LOAD = ["json"]