Language schema 202204 (#3492)

esphome/automation.py

@@ -12,7 +12,7 @@ from esphome.const import (
     CONF_TYPE_ID,
     CONF_TIME,
 )
-from esphome.jsonschema import jschema_extractor
+from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor
 from esphome.util import Registry
 
 
@@ -23,11 +23,10 @@ def maybe_simple_id(*validators):
 def maybe_conf(conf, *validators):
     validator = cv.All(*validators)
 
-    @jschema_extractor("maybe")
+    @schema_extractor("maybe")
     def validate(value):
-        # pylint: disable=comparison-with-callable
+        if value == SCHEMA_EXTRACT:
-        if value == jschema_extractor:
+            return (validator, conf)
-            return validator
 
         if isinstance(value, dict):
             return validator(value)
@@ -111,11 +110,9 @@ def validate_automation(extra_schema=None, extra_validators=None, single=False):
         # This should only happen with invalid configs, but let's have a nice error message.
         return [schema(value)]
 
-    @jschema_extractor("automation")
+    @schema_extractor("automation")
     def validator(value):
-        # hack to get the schema
+        if value == SCHEMA_EXTRACT:
-        # pylint: disable=comparison-with-callable
-        if value == jschema_extractor:
             return schema
 
         value = validator_(value)

esphome/components/light/effects.py

@@ -1,4 +1,4 @@
-from esphome.jsonschema import jschema_extractor
+from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor
 import esphome.codegen as cg
 import esphome.config_validation as cv
 from esphome import automation
@@ -479,11 +479,11 @@ async def addressable_flicker_effect_to_code(config, effect_id):
 
 
 def validate_effects(allowed_effects):
-    @jschema_extractor("effects")
+    @schema_extractor("effects")
     def validator(value):
-        # pylint: disable=comparison-with-callable
+        if value == SCHEMA_EXTRACT:
-        if value == jschema_extractor:
             return (allowed_effects, EFFECTS_REGISTRY)
+
         value = cv.validate_registry("effect", EFFECTS_REGISTRY)(value)
         errors = []
         names = set()

esphome/components/remote_base/__init__.py

@@ -32,7 +32,7 @@ from esphome.const import (
     CONF_LEVEL,
 )
 from esphome.core import coroutine
-from esphome.jsonschema import jschema_extractor
+from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor
 from esphome.util import Registry, SimpleRegistry
 
 AUTO_LOAD = ["binary_sensor"]
@@ -195,14 +195,14 @@ def validate_dumpers(value):
 def validate_triggers(base_schema):
     assert isinstance(base_schema, cv.Schema)
 
-    @jschema_extractor("triggers")
+    @schema_extractor("triggers")
     def validator(config):
         added_keys = {}
         for key, (_, valid) in TRIGGER_REGISTRY.items():
             added_keys[cv.Optional(key)] = valid
         new_schema = base_schema.extend(added_keys)
-        # pylint: disable=comparison-with-callable
+
-        if config == jschema_extractor:
+        if config == SCHEMA_EXTRACT:
             return new_schema
         return new_schema(config)
 

esphome/config_validation.py

@@ -57,11 +57,12 @@ from esphome.core import (
     TimePeriodMinutes,
 )
 from esphome.helpers import list_starts_with, add_class_to_obj
-from esphome.jsonschema import (
+from esphome.schema_extractors import (
-    jschema_list,
+    SCHEMA_EXTRACT,
-    jschema_extractor,
+    schema_extractor_list,
-    jschema_registry,
+    schema_extractor,
-    jschema_typed,
+    schema_extractor_registry,
+    schema_extractor_typed,
 )
 from esphome.util import parse_esphome_version
 from esphome.voluptuous_schema import _Schema
@@ -327,7 +328,7 @@ def boolean(value):
     )
 
 
-@jschema_list
+@schema_extractor_list
 def ensure_list(*validators):
     """Validate this configuration option to be a list.
 
@@ -452,7 +453,11 @@ def validate_id_name(value):
 def use_id(type):
     """Declare that this configuration option should point to an ID with the given type."""
 
+    @schema_extractor("use_id")
     def validator(value):
+        if value == SCHEMA_EXTRACT:
+            return type
+
         check_not_templatable(value)
         if value is None:
             return core.ID(None, is_declaration=False, type=type)
@@ -475,7 +480,11 @@ def declare_id(type):
     If two IDs with the same name exist, a validation error is thrown.
     """
 
+    @schema_extractor("declare_id")
     def validator(value):
+        if value == SCHEMA_EXTRACT:
+            return type
+
         check_not_templatable(value)
         if value is None:
             return core.ID(None, is_declaration=True, type=type)
@@ -494,11 +503,11 @@ def templatable(other_validators):
     """
     schema = Schema(other_validators)
 
-    @jschema_extractor("templatable")
+    @schema_extractor("templatable")
     def validator(value):
-        # pylint: disable=comparison-with-callable
+        if value == SCHEMA_EXTRACT:
-        if value == jschema_extractor:
             return other_validators
+
         if isinstance(value, Lambda):
             return returning_lambda(value)
         if isinstance(other_validators, dict):
@@ -1177,10 +1186,9 @@ def one_of(*values, **kwargs):
     if kwargs:
         raise ValueError
 
-    @jschema_extractor("one_of")
+    @schema_extractor("one_of")
     def validator(value):
-        # pylint: disable=comparison-with-callable
+        if value == SCHEMA_EXTRACT:
-        if value == jschema_extractor:
             return values
 
         if string_:
@@ -1220,10 +1228,9 @@ def enum(mapping, **kwargs):
     assert isinstance(mapping, dict)
     one_of_validator = one_of(*mapping, **kwargs)
 
-    @jschema_extractor("enum")
+    @schema_extractor("enum")
     def validator(value):
-        # pylint: disable=comparison-with-callable
+        if value == SCHEMA_EXTRACT:
-        if value == jschema_extractor:
             return mapping
 
         value = one_of_validator(value)
@@ -1396,7 +1403,7 @@ def extract_keys(schema):
     return keys
 
 
-@jschema_typed
+@schema_extractor_typed
 def typed_schema(schemas, **kwargs):
     """Create a schema that has a key to distinguish between schemas"""
     key = kwargs.pop("key", CONF_TYPE)
@@ -1510,7 +1517,7 @@ def validate_registry_entry(name, registry):
     )
     ignore_keys = extract_keys(base_schema)
 
-    @jschema_registry(registry)
+    @schema_extractor_registry(registry)
     def validator(value):
         if isinstance(value, str):
             value = {value: {}}
@@ -1555,12 +1562,15 @@ def validate_registry(name, registry):
     return ensure_list(validate_registry_entry(name, registry))
 
 
-@jschema_list
 def maybe_simple_value(*validators, **kwargs):
     key = kwargs.pop("key", CONF_VALUE)
     validator = All(*validators)
 
+    @schema_extractor("maybe")
     def validate(value):
+        if value == SCHEMA_EXTRACT:
+            return (validator, key)
+
         if isinstance(value, dict) and key in value:
             return validator(value)
         return validator({key: value})

esphome/schema_extractors.py

@@ -9,9 +9,9 @@ However there is a property to further disable decorator
 impact."""
 
 
-# This is set to true by script/build_jsonschema.py
+# This is set to true by script/build_language_schema.py
 # only, so data is collected (again functionality is not modified)
-EnableJsonSchemaCollect = False
+EnableSchemaExtraction = False
 
 extended_schemas = {}
 list_schemas = {}
@@ -19,9 +19,12 @@ registry_schemas = {}
 hidden_schemas = {}
 typed_schemas = {}
 
+# This key is used to generate schema files of Esphome configuration.
+SCHEMA_EXTRACT = object()
 
-def jschema_extractor(validator_name):
+
-    if EnableJsonSchemaCollect:
+def schema_extractor(validator_name):
+    if EnableSchemaExtraction:
 
         def decorator(func):
             hidden_schemas[repr(func)] = validator_name
@@ -35,8 +38,8 @@ def jschema_extractor(validator_name):
     return dummy
 
 
-def jschema_extended(func):
+def schema_extractor_extended(func):
-    if EnableJsonSchemaCollect:
+    if EnableSchemaExtraction:
 
         def decorate(*args, **kwargs):
             ret = func(*args, **kwargs)
@@ -49,8 +52,8 @@ def jschema_extended(func):
     return func
 
 
-def jschema_list(func):
+def schema_extractor_list(func):
-    if EnableJsonSchemaCollect:
+    if EnableSchemaExtraction:
 
         def decorate(*args, **kwargs):
             ret = func(*args, **kwargs)
@@ -63,8 +66,8 @@ def jschema_list(func):
     return func
 
 
-def jschema_registry(registry):
+def schema_extractor_registry(registry):
-    if EnableJsonSchemaCollect:
+    if EnableSchemaExtraction:
 
         def decorator(func):
             registry_schemas[repr(func)] = registry
@@ -78,8 +81,8 @@ def jschema_registry(registry):
     return dummy
 
 
-def jschema_typed(func):
+def schema_extractor_typed(func):
-    if EnableJsonSchemaCollect:
+    if EnableSchemaExtraction:
 
         def decorate(*args, **kwargs):
             ret = func(*args, **kwargs)

esphome/voluptuous_schema.py

@@ -2,7 +2,7 @@ import difflib
 import itertools
 
 import voluptuous as vol
-from esphome.jsonschema import jschema_extended
+from esphome.schema_extractors import schema_extractor_extended
 
 
 class ExtraKeysInvalid(vol.Invalid):
@@ -203,7 +203,7 @@ class _Schema(vol.Schema):
         self._extra_schemas.append(validator)
         return self
 
-    @jschema_extended
+    @schema_extractor_extended
     # pylint: disable=signature-differs
     def extend(self, *schemas, **kwargs):
         extra = kwargs.pop("extra", None)

script/build_language_schema.py

@@ -1,15 +1,16 @@
 import inspect
 import json
 import argparse
-from operator import truediv
 import os
+import glob
+import re
 import voluptuous as vol
 
-# NOTE: Cannot import other esphome components globally as a modification in jsonschema
+# NOTE: Cannot import other esphome components globally as a modification in vol_schema
 # is needed before modules are loaded
-import esphome.jsonschema as ejs
+import esphome.schema_extractors as ejs
 
-ejs.EnableJsonSchemaCollect = True
+ejs.EnableSchemaExtraction = True
 
 # schema format:
 # Schemas are splitted in several files in json format, one for core stuff, one for each platform (sensor, binary_sensor, etc) and
@@ -60,15 +61,6 @@ solve_registry = []
 
 
 def get_component_names():
-    # return [
-    #     "esphome",
-    #     "esp32",
-    #     "esp8266",
-    #     "logger",
-    #     "sensor",
-    #     "remote_receiver",
-    #     "binary_sensor",
-    # ]
     from esphome.loader import CORE_COMPONENTS_PATH
 
     component_names = ["esphome", "sensor"]
@@ -100,7 +92,7 @@ from esphome import automation
 from esphome import pins
 from esphome.components import remote_base
 from esphome.const import CONF_TYPE
-from esphome.loader import get_platform
+from esphome.loader import get_platform, CORE_COMPONENTS_PATH
 from esphome.helpers import write_file_if_changed
 from esphome.util import Registry
 
@@ -120,9 +112,11 @@ def write_file(name, obj):
 def register_module_schemas(key, module, manifest=None):
     for name, schema in module_schemas(module):
         register_known_schema(key, name, schema)
-    if (
+
-        manifest and manifest.multi_conf and S_CONFIG_SCHEMA in output[key][S_SCHEMAS]
+    if manifest:
-    ):  # not sure about 2nd part of the if, might be useless config (e.g. as3935)
+        # Multi conf should allow list of components
+        # not sure about 2nd part of the if, might be useless config (e.g. as3935)
+        if manifest.multi_conf and S_CONFIG_SCHEMA in output[key][S_SCHEMAS]:
             output[key][S_SCHEMAS][S_CONFIG_SCHEMA]["is_list"] = True
 
 
@@ -265,13 +259,58 @@ def do_esp8266():
 
 
 def fix_remote_receiver():
-    output["remote_receiver.binary_sensor"]["schemas"]["CONFIG_SCHEMA"] = {
+    remote_receiver_schema = output["remote_receiver.binary_sensor"]["schemas"]
+    remote_receiver_schema["CONFIG_SCHEMA"] = {
         "type": "schema",
         "schema": {
             "extends": ["binary_sensor.BINARY_SENSOR_SCHEMA", "core.COMPONENT_SCHEMA"],
-            "config_vars": output["remote_base"]["binary"],
+            "config_vars": output["remote_base"].pop("binary"),
         },
     }
+    remote_receiver_schema["CONFIG_SCHEMA"]["schema"]["config_vars"]["receiver_id"] = {
+        "key": "GeneratedID",
+        "use_id_type": "remote_base::RemoteReceiverBase",
+        "type": "use_id",
+    }
+
+
+def fix_script():
+    output["script"][S_SCHEMAS][S_CONFIG_SCHEMA][S_TYPE] = S_SCHEMA
+    config_schema = output["script"][S_SCHEMAS][S_CONFIG_SCHEMA]
+    config_schema[S_SCHEMA][S_CONFIG_VARS]["id"]["id_type"] = {
+        "class": "script::Script"
+    }
+    config_schema["is_list"] = True
+
+
+def get_logger_tags():
+    pattern = re.compile(r'^static const char \*const TAG = "(\w.*)";', re.MULTILINE)
+    # tags not in components dir
+    tags = [
+        "app",
+        "component",
+        "entity_base",
+        "scheduler",
+        "api.service",
+    ]
+    for x in os.walk(CORE_COMPONENTS_PATH):
+        for y in glob.glob(os.path.join(x[0], "*.cpp")):
+            with open(y, encoding="utf-8") as file:
+                data = file.read()
+                match = pattern.search(data)
+                if match:
+                    tags.append(match.group(1))
+    return tags
+
+
+def add_logger_tags():
+    tags = get_logger_tags()
+    logs = output["logger"]["schemas"]["CONFIG_SCHEMA"]["schema"]["config_vars"][
+        "logs"
+    ]["schema"]["config_vars"]
+    for t in tags:
+        logs[t] = logs["string"].copy()
+    logs.pop("string")
 
 
 def add_referenced_recursive(referenced_schemas, config_var, path, eat_schema=False):
@@ -401,7 +440,7 @@ def shrink():
                     else:
                         print("expected extends here!" + x)
                     arr_s = merge(key_s, arr_s)
-                    if arr_s[S_TYPE] == "enum":
+                    if arr_s[S_TYPE] in ["enum", "typed"]:
                         arr_s.pop(S_SCHEMA)
                 else:
                     arr_s.pop(S_EXTENDS)
@@ -491,14 +530,20 @@ def build_schema():
         if domain not in platforms:
             if manifest.config_schema is not None:
                 core_components[domain] = {}
+                if len(manifest.dependencies) > 0:
+                    core_components[domain]["dependencies"] = manifest.dependencies
             register_module_schemas(domain, manifest.module, manifest)
 
         for platform in platforms:
             platform_manifest = get_platform(domain=platform, platform=domain)
             if platform_manifest is not None:
                 output[platform][S_COMPONENTS][domain] = {}
+                if len(platform_manifest.dependencies) > 0:
+                    output[platform][S_COMPONENTS][domain][
+                        "dependencies"
+                    ] = platform_manifest.dependencies
                 register_module_schemas(
-                    f"{domain}.{platform}", platform_manifest.module
+                    f"{domain}.{platform}", platform_manifest.module, platform_manifest
                 )
 
     # Do registries
@@ -517,6 +562,8 @@ def build_schema():
     do_esp8266()
     do_esp32()
     fix_remote_receiver()
+    fix_script()
+    add_logger_tags()
     shrink()
 
     # aggregate components, so all component info is in same file, otherwise we have dallas.json, dallas.sensor.json, etc.
@@ -585,7 +632,7 @@ def convert_1(schema, config_var, path):
                 assert S_EXTENDS not in config_var
                 if not S_TYPE in config_var:
                     config_var[S_TYPE] = S_SCHEMA
-                assert config_var[S_TYPE] == S_SCHEMA
+                # assert config_var[S_TYPE] == S_SCHEMA
 
                 if S_SCHEMA not in config_var:
                     config_var[S_SCHEMA] = {}
@@ -662,7 +709,7 @@ def convert_1(schema, config_var, path):
     elif repr_schema in ejs.hidden_schemas:
         schema_type = ejs.hidden_schemas[repr_schema]
 
-        data = schema(ejs.jschema_extractor)
+        data = schema(ejs.SCHEMA_EXTRACT)
 
         # enums, e.g. esp32/variant
         if schema_type == "one_of":
@@ -672,8 +719,9 @@ def convert_1(schema, config_var, path):
             config_var[S_TYPE] = "enum"
             config_var["values"] = list(data.keys())
         elif schema_type == "maybe":
-            config_var[S_TYPE] = "maybe"
+            config_var[S_TYPE] = S_SCHEMA
-            config_var["schema"] = convert_config(data, path + "/maybe")["schema"]
+            config_var["maybe"] = data[1]
+            config_var["schema"] = convert_config(data[0], path + "/maybe")["schema"]
         # esphome/on_boot
         elif schema_type == "automation":
             extra_schema = None
@@ -717,8 +765,50 @@ def convert_1(schema, config_var, path):
         elif schema_type == "sensor":
             schema = data
             convert_1(data, config_var, path + "/trigger")
+        elif schema_type == "declare_id":
+            # pylint: disable=protected-access
+            parents = data._parents
+
+            config_var["id_type"] = {
+                "class": str(data.base),
+                "parents": [str(x.base) for x in parents]
+                if isinstance(parents, list)
+                else None,
+            }
+        elif schema_type == "use_id":
+            if inspect.ismodule(data):
+                m_attr_obj = getattr(data, "CONFIG_SCHEMA")
+                use_schema = known_schemas.get(repr(m_attr_obj))
+                if use_schema:
+                    [output_module, output_name] = use_schema[0][1].split(".")
+                    use_id_config = output[output_module][S_SCHEMAS][output_name]
+                    config_var["use_id_type"] = use_id_config["schema"]["config_vars"][
+                        "id"
+                    ]["id_type"]["class"]
+                    config_var[S_TYPE] = "use_id"
+                else:
+                    print("TODO deferred?")
+            else:
+                if isinstance(data, str):
+                    # TODO: Figure out why pipsolar does this
+                    config_var["use_id_type"] = data
+                else:
+                    config_var["use_id_type"] = str(data.base)
+                    config_var[S_TYPE] = "use_id"
         else:
             raise Exception("Unknown extracted schema type")
+    elif config_var.get("key") == "GeneratedID":
+        if path == "i2c/CONFIG_SCHEMA/extL/all/id":
+            config_var["id_type"] = {"class": "i2c::I2CBus", "parents": ["Component"]}
+        elif path == "uart/CONFIG_SCHEMA/val 1/extL/all/id":
+            config_var["id_type"] = {
+                "class": "uart::UARTComponent",
+                "parents": ["Component"],
+            }
+        elif path == "pins/esp32/val 1/id":
+            config_var["id_type"] = "pin"
+        else:
+            raise Exception("Cannot determine id_type for " + path)
 
     elif repr_schema in ejs.registry_schemas:
         solve_registry.append((ejs.registry_schemas[repr_schema], config_var))
@@ -787,7 +877,13 @@ def convert_keys(converted, schema, path):
             result["key"] = "Optional"
         else:
             converted["key"] = "String"
-            converted["key_dump"] = str(k)
+            key_string_match = re.search(
+                r"<function (\w*) at \w*>", str(k), re.IGNORECASE
+            )
+            if key_string_match:
+                converted["key_type"] = key_string_match.group(1)
+            else:
+                converted["key_type"] = str(k)
 
         esphome_core.CORE.data = {
             esphome_core.KEY_CORE: {esphome_core.KEY_TARGET_PLATFORM: "esp8266"}
@@ -808,6 +904,12 @@ def convert_keys(converted, schema, path):
             if base_k in result and base_v == result[base_k]:
                 result.pop(base_k)
         converted["schema"][S_CONFIG_VARS][str(k)] = result
+        if "key" in converted and converted["key"] == "String":
+            config_vars = converted["schema"]["config_vars"]
+            assert len(config_vars) == 1
+            key = list(config_vars.keys())[0]
+            assert key.startswith("<")
+            config_vars["string"] = config_vars.pop(key)
 
 
 build_schema()