From 1e227e8051a288c8c46b17b755a4da8a4aec3893 Mon Sep 17 00:00:00 2001 From: Guillermo Ruffino Date: Sun, 7 Mar 2021 21:05:08 -0300 Subject: [PATCH] Schema dump (#1564) * schema dump idea accept boolean or anything default accept null also for full dicts added some common validators more simple validators support multi_conf better handle automations updates updates handle lists removed not needed class move to own folder generalized for automations lists, etc updates updates clean up clean up fix automations made comment optional basic docs support added more docs fixes docs handling updates updates fix components parent updates updates updates Fix inkplate 6 registration updates Disable logging for vscode add on better handle buses keep extended order as in CONFIGs updates updates updates disable comments moved to scripts/build_jsonschema added configurable decorators path handling fix handle list_schema fixes and cleanup add jschema_extractor to maybe updates lint no schema in git add generated loggers list * lint --- esphome/automation.py | 12 + esphome/components/canbus/__init__.py | 12 +- esphome/components/inkplate6/display.py | 4 +- esphome/components/mcp2515/canbus.py | 2 +- esphome/components/remote_base/__init__.py | 20 +- esphome/config_validation.py | 5 + esphome/const.py | 1 - esphome/jsonschema.py | 90 +++ esphome/voluptuous_schema.py | 2 + script/build_jsonschema.py | 708 +++++++++++++++++++++ setup.py | 69 +- tests/component_tests/conftest.py | 4 +- 12 files changed, 879 insertions(+), 50 deletions(-) create mode 100644 esphome/jsonschema.py create mode 100644 script/build_jsonschema.py diff --git a/esphome/automation.py b/esphome/automation.py index 63e4ce03726..eb6cb025329 100644 --- a/esphome/automation.py +++ b/esphome/automation.py @@ -11,6 +11,7 @@ CONF_TIME, ) from esphome.core import coroutine +from esphome.jsonschema import jschema_extractor from esphome.util import Registry @@ -21,7 +22,12 @@ def maybe_simple_id(*validators): def maybe_conf(conf, *validators): validator = cv.All(*validators) + @jschema_extractor("maybe") def validate(value): + # pylint: disable=comparison-with-callable + if value == jschema_extractor: + return validator + if isinstance(value, dict): return validator(value) with cv.remove_prepend_path([conf]): @@ -103,7 +109,13 @@ def validator_(value): # This should only happen with invalid configs, but let's have a nice error message. return [schema(value)] + @jschema_extractor("automation") def validator(value): + # hack to get the schema + # pylint: disable=comparison-with-callable + if value == jschema_extractor: + return schema + value = validator_(value) if extra_validators is not None: value = cv.Schema([extra_validators])(value) diff --git a/esphome/components/canbus/__init__.py b/esphome/components/canbus/__init__.py index 28501c7a858..1cbebb07da4 100644 --- a/esphome/components/canbus/__init__.py +++ b/esphome/components/canbus/__init__.py @@ -12,7 +12,6 @@ CONF_CANBUS_ID = "canbus_id" CONF_BIT_RATE = "bit_rate" CONF_ON_FRAME = "on_frame" -CONF_CANBUS_SEND = "canbus.send" def validate_id(id_value, id_ext): @@ -59,7 +58,7 @@ def validate_raw_data(value): "1000KBPS": CanSpeed.CAN_1000KBPS, } -CONFIG_SCHEMA = cv.Schema( +CANBUS_SCHEMA = cv.Schema( { cv.GenerateID(): cv.declare_id(CanbusComponent), cv.Required(CONF_CAN_ID): cv.int_range(min=0, max=0x1FFFFFFF), @@ -70,6 +69,13 @@ def validate_raw_data(value): cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(CanbusTrigger), cv.GenerateID(CONF_CAN_ID): cv.int_range(min=0, max=0x1FFFFFFF), cv.Optional(CONF_USE_EXTENDED_ID, default=False): cv.boolean, + cv.Optional(CONF_ON_FRAME): automation.validate_automation( + { + cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(CanbusTrigger), + cv.GenerateID(CONF_CAN_ID): cv.int_range(min=0, max=0x1FFFFFFF), + cv.Optional(CONF_USE_EXTENDED_ID, default=False): cv.boolean, + } + ), } ), } @@ -104,7 +110,7 @@ def register_canbus(var, config): # Actions @automation.register_action( - CONF_CANBUS_SEND, + "canbus.send", canbus_ns.class_("CanbusSendAction", automation.Action), cv.maybe_simple_value( { diff --git a/esphome/components/inkplate6/display.py b/esphome/components/inkplate6/display.py index cd2b5ac51bf..323936d2c42 100644 --- a/esphome/components/inkplate6/display.py +++ b/esphome/components/inkplate6/display.py @@ -87,7 +87,9 @@ CONF_DISPLAY_DATA_7_PIN, default=27 ): pins.internal_gpio_output_pin_schema, } - ).extend(cv.polling_component_schema("5s").extend(i2c.i2c_device_schema(0x48))), + ) + .extend(cv.polling_component_schema("5s")) + .extend(i2c.i2c_device_schema(0x48)), cv.has_at_most_one_key(CONF_PAGES, CONF_LAMBDA), ) diff --git a/esphome/components/mcp2515/canbus.py b/esphome/components/mcp2515/canbus.py index 68e28d45b88..0fc679d17aa 100644 --- a/esphome/components/mcp2515/canbus.py +++ b/esphome/components/mcp2515/canbus.py @@ -26,7 +26,7 @@ "LISTENONLY": McpMode.CANCTRL_REQOP_LISTENONLY, } -CONFIG_SCHEMA = canbus.CONFIG_SCHEMA.extend( +CONFIG_SCHEMA = canbus.CANBUS_SCHEMA.extend( { cv.GenerateID(): cv.declare_id(mcp2515), cv.Optional(CONF_CLOCK, default="8MHZ"): cv.enum(CAN_CLOCK, upper=True), diff --git a/esphome/components/remote_base/__init__.py b/esphome/components/remote_base/__init__.py index 7e81daa1e1a..96579c05bba 100644 --- a/esphome/components/remote_base/__init__.py +++ b/esphome/components/remote_base/__init__.py @@ -29,6 +29,7 @@ CONF_RC_CODE_2, ) from esphome.core import coroutine +from esphome.jsonschema import jschema_extractor from esphome.util import Registry, SimpleRegistry AUTO_LOAD = ["binary_sensor"] @@ -123,13 +124,16 @@ def validate_repeat(value): return validate_repeat({CONF_TIMES: value}) +BASE_REMOTE_TRANSMITTER_SCHEMA = cv.Schema( + { + cv.GenerateID(CONF_TRANSMITTER_ID): cv.use_id(RemoteTransmitterBase), + cv.Optional(CONF_REPEAT): validate_repeat, + } +) + + def register_action(name, type_, schema): - validator = templatize(schema).extend( - { - cv.GenerateID(CONF_TRANSMITTER_ID): cv.use_id(RemoteTransmitterBase), - cv.Optional(CONF_REPEAT): validate_repeat, - } - ) + validator = templatize(schema).extend(BASE_REMOTE_TRANSMITTER_SCHEMA) registerer = automation.register_action( f"remote_transmitter.transmit_{name}", type_, validator ) @@ -190,11 +194,15 @@ def validate_dumpers(value): def validate_triggers(base_schema): assert isinstance(base_schema, cv.Schema) + @jschema_extractor("triggers") def validator(config): added_keys = {} for key, (_, valid) in TRIGGER_REGISTRY.items(): added_keys[cv.Optional(key)] = valid new_schema = base_schema.extend(added_keys) + # pylint: disable=comparison-with-callable + if config == jschema_extractor: + return new_schema return new_schema(config) return validator diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 46cc1fad50c..4a65c593798 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -46,6 +46,7 @@ TimePeriodMinutes, ) from esphome.helpers import list_starts_with, add_class_to_obj +from esphome.jsonschema import jschema_composite, jschema_registry, jschema_typed from esphome.voluptuous_schema import _Schema from esphome.yaml_util import make_data_base @@ -306,6 +307,7 @@ def boolean(value): ) +@jschema_composite def ensure_list(*validators): """Validate this configuration option to be a list. @@ -1341,6 +1343,7 @@ def extract_keys(schema): return keys +@jschema_typed def typed_schema(schemas, **kwargs): """Create a schema that has a key to distinguish between schemas""" key = kwargs.pop("key", CONF_TYPE) @@ -1442,6 +1445,7 @@ def validate_registry_entry(name, registry): ) ignore_keys = extract_keys(base_schema) + @jschema_registry(registry) def validator(value): if isinstance(value, str): value = {value: {}} @@ -1488,6 +1492,7 @@ def validate_registry(name, registry): return ensure_list(validate_registry_entry(name, registry)) +@jschema_composite def maybe_simple_value(*validators, **kwargs): key = kwargs.pop("key", CONF_VALUE) validator = All(*validators) diff --git a/esphome/const.py b/esphome/const.py index af5961be888..5d735698bc7 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -180,7 +180,6 @@ CONF_ENTITY_ID = "entity_id" CONF_ESP8266_RESTORE_FROM_FLASH = "esp8266_restore_from_flash" CONF_ESPHOME = "esphome" -CONF_ESPHOME_CORE_VERSION = "esphome_core_version" CONF_EVENT = "event" CONF_EXPIRE_AFTER = "expire_after" CONF_EXTERNAL_VCC = "external_vcc" diff --git a/esphome/jsonschema.py b/esphome/jsonschema.py new file mode 100644 index 00000000000..25d85ed58ff --- /dev/null +++ b/esphome/jsonschema.py @@ -0,0 +1,90 @@ +# These are a helper decorators to help get schema from some +# components which uses volutuous in a way where validation +# is hidden in local functions + +# These decorators should not modify at all what the functions +# originally do. +# +# However there is a property to further disable decorator +# impat. +# +# This is set to true by script/build_jsonschema.py +# only, so data is collected (again functionality is not modified) +EnableJsonSchemaCollect = False + +extended_schemas = {} +list_schemas = {} +registry_schemas = {} +hidden_schemas = {} +typed_schemas = {} + + +def jschema_extractor(validator_name): + if EnableJsonSchemaCollect: + + def decorator(func): + hidden_schemas[str(func)] = validator_name + return func + + return decorator + + def dummy(f): + return f + + return dummy + + +def jschema_extended(func): + if EnableJsonSchemaCollect: + + def decorate(*args, **kwargs): + ret = func(*args, **kwargs) + assert len(args) == 2 + extended_schemas[str(ret)] = args + return ret + + return decorate + + return func + + +def jschema_composite(func): + if EnableJsonSchemaCollect: + + def decorate(*args, **kwargs): + ret = func(*args, **kwargs) + # args length might be 2, but 2nd is always validator + list_schemas[str(ret)] = args + return ret + + return decorate + + return func + + +def jschema_registry(registry): + if EnableJsonSchemaCollect: + + def decorator(func): + registry_schemas[str(func)] = registry + return func + + return decorator + + def dummy(f): + return f + + return dummy + + +def jschema_typed(func): + if EnableJsonSchemaCollect: + + def decorate(*args, **kwargs): + ret = func(*args, **kwargs) + typed_schemas[str(ret)] = (args, kwargs) + return ret + + return decorate + + return func diff --git a/esphome/voluptuous_schema.py b/esphome/voluptuous_schema.py index 86c20510aae..0fdae423cf2 100644 --- a/esphome/voluptuous_schema.py +++ b/esphome/voluptuous_schema.py @@ -2,6 +2,7 @@ import itertools import voluptuous as vol +from esphome.jsonschema import jschema_extended class ExtraKeysInvalid(vol.Invalid): @@ -202,6 +203,7 @@ def add_extra(self, validator): self._extra_schemas.append(validator) return self + @jschema_extended # pylint: disable=signature-differs def extend(self, *schemas, **kwargs): extra = kwargs.pop("extra", None) diff --git a/script/build_jsonschema.py b/script/build_jsonschema.py new file mode 100644 index 00000000000..e179c95541c --- /dev/null +++ b/script/build_jsonschema.py @@ -0,0 +1,708 @@ +#!/usr/bin/env python3 + +import json +import argparse +import os +import re +from pathlib import Path +import voluptuous as vol + +# NOTE: Cannot import other esphome components globally as a modification in jsonschema +# is needed before modules are loaded +import esphome.jsonschema as ejs + +ejs.EnableJsonSchemaCollect = True + +DUMP_COMMENTS = False + +JSC_ACTION = "automation.ACTION_REGISTRY" +JSC_ALLOF = "allOf" +JSC_ANYOF = "anyOf" +JSC_COMMENT = "$comment" +JSC_CONDITION = "automation.CONDITION_REGISTRY" +JSC_DESCRIPTION = "description" +JSC_ONEOF = "oneOf" +JSC_PROPERTIES = "properties" +JSC_REF = "$ref" +SIMPLE_AUTOMATION = "simple_automation" + +schema_names = {} +schema_registry = {} +components = {} +modules = {} +registries = [] +pending_refs = [] + +definitions = {} +base_props = {} + + +parser = argparse.ArgumentParser() +parser.add_argument( + "--output", default="esphome.json", help="Output filename", type=os.path.abspath +) + +args = parser.parse_args() + + +def get_ref(definition): + return {JSC_REF: "#/definitions/" + definition} + + +def is_ref(jschema): + return isinstance(jschema, dict) and JSC_REF in jschema + + +def unref(jschema): + return definitions[jschema[JSC_REF][len("#/definitions/") :]] + + +def add_definition_array_or_single_object(ref): + return {JSC_ANYOF: [{"type": "array", "items": ref}, ref]} + + +def add_core(): + from esphome.core_config import CONFIG_SCHEMA + + base_props["esphome"] = get_jschema("esphome", CONFIG_SCHEMA.schema) + + +def add_buses(): + # uart + from esphome.components.uart import UART_DEVICE_SCHEMA + + get_jschema("uart_bus", UART_DEVICE_SCHEMA) + + # spi + from esphome.components.spi import spi_device_schema + + get_jschema("spi_bus", spi_device_schema(False)) + + # i2c + from esphome.components.i2c import i2c_device_schema + + get_jschema("i2c_bus", i2c_device_schema(None)) + + +def add_registries(): + for domain, module in modules.items(): + add_module_registries(domain, module) + + +def add_module_registries(domain, module): + from esphome.util import Registry + + for c in dir(module): + m = getattr(module, c) + if isinstance(m, Registry): + add_registry(domain + "." + c, m) + + +def add_registry(registry_name, registry): + validators = [] + registries.append((registry, registry_name)) + for name in registry.keys(): + schema = get_jschema(str(name), registry[name].schema, create_return_ref=False) + if not schema: + schema = {"type": "string"} + o_schema = {"type": "object", JSC_PROPERTIES: {name: schema}} + validators.append(o_schema) + definitions[registry_name] = {JSC_ANYOF: validators} + + +def get_registry_ref(registry): + # we don't know yet + ref = {JSC_REF: "pending"} + pending_refs.append((ref, registry)) + return ref + + +def solve_pending_refs(): + for ref, registry in pending_refs: + for registry_match, name in registries: + if registry == registry_match: + ref[JSC_REF] = "#/definitions/" + name + + +def add_module_schemas(name, module): + import esphome.config_validation as cv + + for c in dir(module): + v = getattr(module, c) + if isinstance(v, cv.Schema): + get_jschema(name + "." + c, v) + + +def get_dirs(): + from esphome.config import CORE_COMPONENTS_PATH + + dir_names = [ + d + for d in os.listdir(CORE_COMPONENTS_PATH) + if not d.startswith("__") + and os.path.isdir(os.path.join(CORE_COMPONENTS_PATH, d)) + ] + return dir_names + + +def get_logger_tags(): + from esphome.config import CORE_COMPONENTS_PATH + import glob + + pattern = re.compile(r'^static const char(\*\s|\s\*)TAG = "(\w.*)";', re.MULTILINE) + tags = [ + "app", + "component", + "esphal", + "helpers", + "preferences", + "scheduler", + "api.service", + ] + for x in os.walk(CORE_COMPONENTS_PATH): + for y in glob.glob(os.path.join(x[0], "*.cpp")): + with open(y, "r") as file: + data = file.read() + match = pattern.search(data) + if match: + tags.append(match.group(2)) + return tags + + +def load_components(): + import esphome.config_validation as cv + from esphome.config import get_component + + modules["cv"] = cv + from esphome import automation + + modules["automation"] = automation + + for domain in get_dirs(): + components[domain] = get_component(domain) + modules[domain] = components[domain].module + + +def add_components(): + from esphome.config import get_platform + + for domain, c in components.items(): + if c.is_platform_component: + # this is a platform_component, e.g. binary_sensor + platform_schema = [ + { + "type": "object", + "properties": {"platform": {"type": "string"}}, + } + ] + if domain != "output" and domain != "display": + # output bases are either FLOAT or BINARY so don't add common base for this + # display bases are either simple or FULL so don't add common base for this + platform_schema = [ + {"$ref": f"#/definitions/{domain}.{domain.upper()}_SCHEMA"} + ] + platform_schema + + base_props[domain] = {"type": "array", "items": {"allOf": platform_schema}} + + add_module_registries(domain, c.module) + add_module_schemas(domain, c.module) + + # need first to iterate all platforms then iteate components + # a platform component can have other components as properties, + # e.g. climate components usually have a temperature sensor + + for domain, c in components.items(): + if (c.config_schema is not None) or c.is_platform_component: + if c.is_platform_component: + platform_schema = base_props[domain]["items"]["allOf"] + for platform in get_dirs(): + p = get_platform(domain, platform) + if p is not None: + # this is a platform element, e.g. + # - platform: gpio + schema = get_jschema( + domain + "-" + platform, + p.config_schema, + create_return_ref=False, + ) + if ( + schema + ): # for invalid schemas, None is returned thus is deprecated + platform_schema.append( + { + "if": { + JSC_PROPERTIES: { + "platform": {"const": platform} + } + }, + "then": schema, + } + ) + + elif c.config_schema is not None: + # adds root components which are not platforms, e.g. api: logger: + if c.is_multi_conf: + schema = get_jschema(domain, c.config_schema) + schema = add_definition_array_or_single_object(schema) + else: + schema = get_jschema(domain, c.config_schema, False) + base_props[domain] = schema + + +def get_automation_schema(name, vschema): + from esphome.automation import AUTOMATION_SCHEMA + + # ensure SIMPLE_AUTOMATION + if SIMPLE_AUTOMATION not in definitions: + simple_automation = add_definition_array_or_single_object(get_ref(JSC_ACTION)) + simple_automation[JSC_ANYOF].append( + get_jschema(AUTOMATION_SCHEMA.__module__, AUTOMATION_SCHEMA) + ) + + definitions[schema_names[str(AUTOMATION_SCHEMA)]][JSC_PROPERTIES][ + "then" + ] = add_definition_array_or_single_object(get_ref(JSC_ACTION)) + definitions[SIMPLE_AUTOMATION] = simple_automation + + extra_vschema = None + if AUTOMATION_SCHEMA == ejs.extended_schemas[str(vschema)][0]: + extra_vschema = ejs.extended_schemas[str(vschema)][1] + + if not extra_vschema: + return get_ref(SIMPLE_AUTOMATION) + + # add then property + extra_jschema = get_jschema(name, extra_vschema, False) + + if is_ref(extra_jschema): + return extra_jschema + + if not JSC_PROPERTIES in extra_jschema: + # these are interval: and exposure_notifications, featuring automations a component + extra_jschema[JSC_ALLOF][0][JSC_PROPERTIES][ + "then" + ] = add_definition_array_or_single_object(get_ref(JSC_ACTION)) + ref = create_ref(name, extra_vschema, extra_jschema) + return add_definition_array_or_single_object(ref) + + # automations can be either + # * a single action, + # * an array of action, + # * an object with automation's schema and a then key + # with again a single action or an array of actions + + extra_jschema[JSC_PROPERTIES]["then"] = add_definition_array_or_single_object( + get_ref(JSC_ACTION) + ) + jschema = add_definition_array_or_single_object(get_ref(JSC_ACTION)) + jschema[JSC_ANYOF].append(extra_jschema) + + return create_ref(name, extra_vschema, jschema) + + +def get_entry(parent_key, vschema): + from esphome.voluptuous_schema import _Schema as schema_type + + entry = {} + # annotate schema validator info + if DUMP_COMMENTS: + entry[JSC_COMMENT] = "entry: " + parent_key + "/" + str(vschema) + + if isinstance(vschema, list): + ref = get_jschema(parent_key + "[]", vschema[0]) + entry = {"type": "array", "items": ref} + elif isinstance(vschema, schema_type) and hasattr(vschema, "schema"): + entry = get_jschema(parent_key, vschema, False) + elif hasattr(vschema, "validators"): + entry = get_jschema(parent_key, vschema, False) + elif vschema in schema_registry: + entry = schema_registry[vschema].copy() + elif str(vschema) in ejs.registry_schemas: + entry = get_registry_ref(ejs.registry_schemas[str(vschema)]) + elif str(vschema) in ejs.list_schemas: + ref = get_jschema(parent_key, ejs.list_schemas[str(vschema)][0]) + entry = {JSC_ANYOF: [ref, {"type": "array", "items": ref}]} + + elif str(vschema) in ejs.typed_schemas: + schema_types = [{"type": "object", "properties": {"type": {"type": "string"}}}] + entry = {"allOf": schema_types} + for schema_key, vschema_type in ejs.typed_schemas[str(vschema)][0][0].items(): + schema_types.append( + { + "if": {"properties": {"type": {"const": schema_key}}}, + "then": get_jschema(f"{parent_key}-{schema_key}", vschema_type), + } + ) + + elif str(vschema) in ejs.hidden_schemas: + # get the schema from the automation schema + type = ejs.hidden_schemas[str(vschema)] + inner_vschema = vschema(ejs.jschema_extractor) + if type == "automation": + entry = get_automation_schema(parent_key, inner_vschema) + elif type == "maybe": + entry = get_jschema(parent_key, inner_vschema) + else: + raise ValueError("Unknown extracted schema type") + elif str(vschema).startswith(" str: CORE.config_path = path