Schema dump (#1564)

* schema dump idea

accept boolean or anything default

accept null also for full dicts

added some common validators

more simple validators

support multi_conf

better handle automations

updates

updates

handle lists

removed not needed class

move to own folder

generalized for automations lists, etc

updates

updates

clean up

clean up

fix automations

made comment optional

basic docs support

added more docs

fixes docs handling

updates

updates

fix components parent

updates

updates

updates

Fix inkplate 6 registration

updates

Disable logging for vscode add on

better handle buses

keep extended order as in CONFIGs

updates

updates

updates

disable comments

moved to scripts/build_jsonschema

added configurable decorators

path handling

fix handle list_schema

fixes and cleanup

add jschema_extractor to maybe

updates

lint

no schema in git

add generated loggers list

* lint

esphome/automation.py

@@ -11,6 +11,7 @@ from esphome.const import (
     CONF_TIME,
 )
 from esphome.core import coroutine
+from esphome.jsonschema import jschema_extractor
 from esphome.util import Registry
 
 
@@ -21,7 +22,12 @@ def maybe_simple_id(*validators):
 def maybe_conf(conf, *validators):
     validator = cv.All(*validators)
 
+    @jschema_extractor("maybe")
     def validate(value):
+        # pylint: disable=comparison-with-callable
+        if value == jschema_extractor:
+            return validator
+
         if isinstance(value, dict):
             return validator(value)
         with cv.remove_prepend_path([conf]):
@@ -103,7 +109,13 @@ def validate_automation(extra_schema=None, extra_validators=None, single=False):
         # This should only happen with invalid configs, but let's have a nice error message.
         return [schema(value)]
 
+    @jschema_extractor("automation")
     def validator(value):
+        # hack to get the schema
+        # pylint: disable=comparison-with-callable
+        if value == jschema_extractor:
+            return schema
+
         value = validator_(value)
         if extra_validators is not None:
             value = cv.Schema([extra_validators])(value)

esphome/components/canbus/__init__.py

@@ -12,7 +12,6 @@ CONF_USE_EXTENDED_ID = "use_extended_id"
 CONF_CANBUS_ID = "canbus_id"
 CONF_BIT_RATE = "bit_rate"
 CONF_ON_FRAME = "on_frame"
-CONF_CANBUS_SEND = "canbus.send"
 
 
 def validate_id(id_value, id_ext):
@@ -59,7 +58,7 @@ CAN_SPEEDS = {
     "1000KBPS": CanSpeed.CAN_1000KBPS,
 }
 
-CONFIG_SCHEMA = cv.Schema(
+CANBUS_SCHEMA = cv.Schema(
     {
         cv.GenerateID(): cv.declare_id(CanbusComponent),
         cv.Required(CONF_CAN_ID): cv.int_range(min=0, max=0x1FFFFFFF),
@@ -70,6 +69,13 @@ CONFIG_SCHEMA = cv.Schema(
                 cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(CanbusTrigger),
                 cv.GenerateID(CONF_CAN_ID): cv.int_range(min=0, max=0x1FFFFFFF),
                 cv.Optional(CONF_USE_EXTENDED_ID, default=False): cv.boolean,
+                cv.Optional(CONF_ON_FRAME): automation.validate_automation(
+                    {
+                        cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(CanbusTrigger),
+                        cv.GenerateID(CONF_CAN_ID): cv.int_range(min=0, max=0x1FFFFFFF),
+                        cv.Optional(CONF_USE_EXTENDED_ID, default=False): cv.boolean,
+                    }
+                ),
             }
         ),
     }
@@ -104,7 +110,7 @@ def register_canbus(var, config):
 
 # Actions
 @automation.register_action(
-    CONF_CANBUS_SEND,
+    "canbus.send",
     canbus_ns.class_("CanbusSendAction", automation.Action),
     cv.maybe_simple_value(
         {

esphome/components/inkplate6/display.py

@@ -87,7 +87,9 @@ CONFIG_SCHEMA = cv.All(
                 CONF_DISPLAY_DATA_7_PIN, default=27
             ): pins.internal_gpio_output_pin_schema,
         }
-    ).extend(cv.polling_component_schema("5s").extend(i2c.i2c_device_schema(0x48))),
+    )
+    .extend(cv.polling_component_schema("5s"))
+    .extend(i2c.i2c_device_schema(0x48)),
     cv.has_at_most_one_key(CONF_PAGES, CONF_LAMBDA),
 )
 

esphome/components/mcp2515/canbus.py

@@ -26,7 +26,7 @@ MCP_MODE = {
     "LISTENONLY": McpMode.CANCTRL_REQOP_LISTENONLY,
 }
 
-CONFIG_SCHEMA = canbus.CONFIG_SCHEMA.extend(
+CONFIG_SCHEMA = canbus.CANBUS_SCHEMA.extend(
     {
         cv.GenerateID(): cv.declare_id(mcp2515),
         cv.Optional(CONF_CLOCK, default="8MHZ"): cv.enum(CAN_CLOCK, upper=True),

esphome/components/remote_base/__init__.py

@@ -29,6 +29,7 @@ from esphome.const import (
     CONF_RC_CODE_2,
 )
 from esphome.core import coroutine
+from esphome.jsonschema import jschema_extractor
 from esphome.util import Registry, SimpleRegistry
 
 AUTO_LOAD = ["binary_sensor"]
@@ -123,13 +124,16 @@ def validate_repeat(value):
     return validate_repeat({CONF_TIMES: value})
 
 
+BASE_REMOTE_TRANSMITTER_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(CONF_TRANSMITTER_ID): cv.use_id(RemoteTransmitterBase),
+        cv.Optional(CONF_REPEAT): validate_repeat,
+    }
+)
+
+
 def register_action(name, type_, schema):
-    validator = templatize(schema).extend(
-        {
-            cv.GenerateID(CONF_TRANSMITTER_ID): cv.use_id(RemoteTransmitterBase),
-            cv.Optional(CONF_REPEAT): validate_repeat,
-        }
-    )
+    validator = templatize(schema).extend(BASE_REMOTE_TRANSMITTER_SCHEMA)
     registerer = automation.register_action(
         f"remote_transmitter.transmit_{name}", type_, validator
     )
@@ -190,11 +194,15 @@ def validate_dumpers(value):
 def validate_triggers(base_schema):
     assert isinstance(base_schema, cv.Schema)
 
+    @jschema_extractor("triggers")
     def validator(config):
         added_keys = {}
         for key, (_, valid) in TRIGGER_REGISTRY.items():
             added_keys[cv.Optional(key)] = valid
         new_schema = base_schema.extend(added_keys)
+        # pylint: disable=comparison-with-callable
+        if config == jschema_extractor:
+            return new_schema
         return new_schema(config)
 
     return validator

esphome/config_validation.py

@@ -46,6 +46,7 @@ from esphome.core import (
     TimePeriodMinutes,
 )
 from esphome.helpers import list_starts_with, add_class_to_obj
+from esphome.jsonschema import jschema_composite, jschema_registry, jschema_typed
 from esphome.voluptuous_schema import _Schema
 from esphome.yaml_util import make_data_base
 
@@ -306,6 +307,7 @@ def boolean(value):
     )
 
 
+@jschema_composite
 def ensure_list(*validators):
     """Validate this configuration option to be a list.
 
@@ -1341,6 +1343,7 @@ def extract_keys(schema):
     return keys
 
 
+@jschema_typed
 def typed_schema(schemas, **kwargs):
     """Create a schema that has a key to distinguish between schemas"""
     key = kwargs.pop("key", CONF_TYPE)
@@ -1442,6 +1445,7 @@ def validate_registry_entry(name, registry):
     )
     ignore_keys = extract_keys(base_schema)
 
+    @jschema_registry(registry)
     def validator(value):
         if isinstance(value, str):
             value = {value: {}}
@@ -1488,6 +1492,7 @@ def validate_registry(name, registry):
     return ensure_list(validate_registry_entry(name, registry))
 
 
+@jschema_composite
 def maybe_simple_value(*validators, **kwargs):
     key = kwargs.pop("key", CONF_VALUE)
     validator = All(*validators)

esphome/const.py

@@ -180,7 +180,6 @@ CONF_ENERGY = "energy"
 CONF_ENTITY_ID = "entity_id"
 CONF_ESP8266_RESTORE_FROM_FLASH = "esp8266_restore_from_flash"
 CONF_ESPHOME = "esphome"
-CONF_ESPHOME_CORE_VERSION = "esphome_core_version"
 CONF_EVENT = "event"
 CONF_EXPIRE_AFTER = "expire_after"
 CONF_EXTERNAL_VCC = "external_vcc"

esphome/jsonschema.py

@@ -0,0 +1,90 @@
+# These are a helper decorators to help get schema from some
+# components which uses volutuous in a way where validation
+# is hidden in local functions
+
+# These decorators should not modify at all what the functions
+# originally do.
+#
+# However there is a property to further disable decorator
+# impat.
+#
+# This is set to true by script/build_jsonschema.py
+# only, so data is collected (again functionality is not modified)
+EnableJsonSchemaCollect = False
+
+extended_schemas = {}
+list_schemas = {}
+registry_schemas = {}
+hidden_schemas = {}
+typed_schemas = {}
+
+
+def jschema_extractor(validator_name):
+    if EnableJsonSchemaCollect:
+
+        def decorator(func):
+            hidden_schemas[str(func)] = validator_name
+            return func
+
+        return decorator
+
+    def dummy(f):
+        return f
+
+    return dummy
+
+
+def jschema_extended(func):
+    if EnableJsonSchemaCollect:
+
+        def decorate(*args, **kwargs):
+            ret = func(*args, **kwargs)
+            assert len(args) == 2
+            extended_schemas[str(ret)] = args
+            return ret
+
+        return decorate
+
+    return func
+
+
+def jschema_composite(func):
+    if EnableJsonSchemaCollect:
+
+        def decorate(*args, **kwargs):
+            ret = func(*args, **kwargs)
+            # args length might be 2, but 2nd is always validator
+            list_schemas[str(ret)] = args
+            return ret
+
+        return decorate
+
+    return func
+
+
+def jschema_registry(registry):
+    if EnableJsonSchemaCollect:
+
+        def decorator(func):
+            registry_schemas[str(func)] = registry
+            return func
+
+        return decorator
+
+    def dummy(f):
+        return f
+
+    return dummy
+
+
+def jschema_typed(func):
+    if EnableJsonSchemaCollect:
+
+        def decorate(*args, **kwargs):
+            ret = func(*args, **kwargs)
+            typed_schemas[str(ret)] = (args, kwargs)
+            return ret
+
+        return decorate
+
+    return func

esphome/voluptuous_schema.py

@@ -2,6 +2,7 @@ import difflib
 import itertools
 
 import voluptuous as vol
+from esphome.jsonschema import jschema_extended
 
 
 class ExtraKeysInvalid(vol.Invalid):
@@ -202,6 +203,7 @@ class _Schema(vol.Schema):
         self._extra_schemas.append(validator)
         return self
 
+    @jschema_extended
     # pylint: disable=signature-differs
     def extend(self, *schemas, **kwargs):
         extra = kwargs.pop("extra", None)

setup.py

@@ -6,49 +6,50 @@ from setuptools import setup, find_packages
 
 from esphome import const
 
-PROJECT_NAME = 'esphome'
-PROJECT_PACKAGE_NAME = 'esphome'
-PROJECT_LICENSE = 'MIT'
-PROJECT_AUTHOR = 'ESPHome'
-PROJECT_COPYRIGHT = '2019, ESPHome'
-PROJECT_URL = 'https://esphome.io/'
-PROJECT_EMAIL = 'contact@esphome.io'
+PROJECT_NAME = "esphome"
+PROJECT_PACKAGE_NAME = "esphome"
+PROJECT_LICENSE = "MIT"
+PROJECT_AUTHOR = "ESPHome"
+PROJECT_COPYRIGHT = "2019, ESPHome"
+PROJECT_URL = "https://esphome.io/"
+PROJECT_EMAIL = "contact@esphome.io"
 
-PROJECT_GITHUB_USERNAME = 'esphome'
-PROJECT_GITHUB_REPOSITORY = 'esphome'
+PROJECT_GITHUB_USERNAME = "esphome"
+PROJECT_GITHUB_REPOSITORY = "esphome"
 
-PYPI_URL = 'https://pypi.python.org/pypi/{}'.format(PROJECT_PACKAGE_NAME)
-GITHUB_PATH = '{}/{}'.format(PROJECT_GITHUB_USERNAME, PROJECT_GITHUB_REPOSITORY)
-GITHUB_URL = 'https://github.com/{}'.format(GITHUB_PATH)
+PYPI_URL = "https://pypi.python.org/pypi/{}".format(PROJECT_PACKAGE_NAME)
+GITHUB_PATH = "{}/{}".format(PROJECT_GITHUB_USERNAME, PROJECT_GITHUB_REPOSITORY)
+GITHUB_URL = "https://github.com/{}".format(GITHUB_PATH)
 
-DOWNLOAD_URL = '{}/archive/v{}.zip'.format(GITHUB_URL, const.__version__)
+DOWNLOAD_URL = "{}/archive/v{}.zip".format(GITHUB_URL, const.__version__)
 
 here = os.path.abspath(os.path.dirname(__file__))
 
-with open(os.path.join(here, 'requirements.txt')) as requirements_txt:
+with open(os.path.join(here, "requirements.txt")) as requirements_txt:
     REQUIRES = requirements_txt.read().splitlines()
 
-with open(os.path.join(here, 'README.md')) as readme:
+with open(os.path.join(here, "README.md")) as readme:
     LONG_DESCRIPTION = readme.read()
 
 # If you have problems importing platformio and esptool as modules you can set
 # $ESPHOME_USE_SUBPROCESS to make ESPHome call their executables instead.
 # This means they have to be in your $PATH.
-if 'ESPHOME_USE_SUBPROCESS' in os.environ:
+if "ESPHOME_USE_SUBPROCESS" in os.environ:
     # Remove platformio and esptool from requirements
     REQUIRES = [
-        req for req in REQUIRES
-        if not any(req.startswith(prefix) for prefix in ['platformio', 'esptool'])
+        req
+        for req in REQUIRES
+        if not any(req.startswith(prefix) for prefix in ["platformio", "esptool"])
     ]
 
 CLASSIFIERS = [
-    'Environment :: Console',
-    'Intended Audience :: Developers',
-    'Intended Audience :: End Users/Desktop',
-    'License :: OSI Approved :: MIT License',
-    'Programming Language :: C++',
-    'Programming Language :: Python :: 3',
-    'Topic :: Home Automation',
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: End Users/Desktop",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: C++",
+    "Programming Language :: Python :: 3",
+    "Topic :: Home Automation",
 ]
 
 setup(
@@ -68,18 +69,14 @@ setup(
     author_email=PROJECT_EMAIL,
     description="Make creating custom firmwares for ESP32/ESP8266 super easy.",
     long_description=LONG_DESCRIPTION,
-    long_description_content_type='text/markdown',
+    long_description_content_type="text/markdown",
     include_package_data=True,
     zip_safe=False,
-    platforms='any',
-    test_suite='tests',
-    python_requires='>=3.6,<4.0',
+    platforms="any",
+    test_suite="tests",
+    python_requires=">=3.6,<4.0",
     install_requires=REQUIRES,
-    keywords=['home', 'automation'],
-    entry_points={
-        'console_scripts': [
-            'esphome = esphome.__main__:main'
-        ]
-    },
-    packages=find_packages(include="esphome.*")
+    keywords=["home", "automation"],
+    entry_points={"console_scripts": ["esphome = esphome.__main__:main"]},
+    packages=find_packages(include="esphome.*"),
 )

tests/component_tests/conftest.py

@@ -1,4 +1,4 @@
-""" Fixtures for component tests """
+"""Fixtures for component tests."""
 
 import pytest
 
@@ -9,7 +9,7 @@ from esphome.__main__ import generate_cpp_contents
 
 @pytest.fixture
 def generate_main():
-    """ Generates the C++ main.cpp file and returns it in string form """
+    """Generates the C++ main.cpp file and returns it in string form."""
 
     def generator(path: str) -> str:
         CORE.config_path = path