[core] Add 'packages' to the config (#5140)

* Initial

* Merge dicts and lists

* feedback

* Move to homeassistant

* feedback

* increase_coverage

* kick_the_hound

homeassistant/bootstrap.py

@@ -395,6 +395,10 @@ def async_from_config_dict(config: Dict[str, Any],
     if not loader.PREPARED:
         yield from hass.loop.run_in_executor(None, loader.prepare, hass)
 
+    # Merge packages
+    conf_util.merge_packages_config(
+        config, core_config.get(conf_util.CONF_PACKAGES, {}))
+
     # Make a copy because we are mutating it.
     # Use OrderedDict in case original one was one.
     # Convert values to dictionaries if they are None

homeassistant/components/script.py

@@ -42,7 +42,7 @@ _SCRIPT_ENTRY_SCHEMA = vol.Schema({
 })
 
 CONFIG_SCHEMA = vol.Schema({
-    vol.Required(DOMAIN): vol.Schema({cv.slug: _SCRIPT_ENTRY_SCHEMA})
+    DOMAIN: vol.Schema({cv.slug: _SCRIPT_ENTRY_SCHEMA})
 }, extra=vol.ALLOW_EXTRA)
 
 SCRIPT_SERVICE_SCHEMA = vol.Schema(dict)

homeassistant/config.py

@@ -1,22 +1,23 @@
 """Module to help with parsing and generating configuration files."""
 import asyncio
+from collections import OrderedDict
 import logging
 import os
 import shutil
 from types import MappingProxyType
-
 # pylint: disable=unused-import
 from typing import Any, Tuple  # NOQA
 
 import voluptuous as vol
 
 from homeassistant.const import (
-    CONF_LATITUDE, CONF_LONGITUDE, CONF_NAME, CONF_UNIT_SYSTEM,
+    CONF_LATITUDE, CONF_LONGITUDE, CONF_NAME, CONF_PACKAGES, CONF_UNIT_SYSTEM,
     CONF_TIME_ZONE, CONF_CUSTOMIZE, CONF_ELEVATION, CONF_UNIT_SYSTEM_METRIC,
     CONF_UNIT_SYSTEM_IMPERIAL, CONF_TEMPERATURE_UNIT, TEMP_CELSIUS,
     __version__)
-from homeassistant.core import valid_entity_id
+from homeassistant.core import valid_entity_id, DOMAIN as CONF_CORE
 from homeassistant.exceptions import HomeAssistantError
+from homeassistant.loader import get_component
 from homeassistant.util.yaml import load_yaml
 import homeassistant.helpers.config_validation as cv
 from homeassistant.helpers.entity import set_customize
@@ -101,6 +102,11 @@ def _valid_customize(value):
     return value
 
 
+PACKAGES_CONFIG_SCHEMA = vol.Schema({
+    cv.slug: vol.Schema(  # Package names are slugs
+        {cv.slug: vol.Any(dict, list)})  # Only slugs for component names
+})
+
 CORE_CONFIG_SCHEMA = vol.Schema({
     CONF_NAME: vol.Coerce(str),
     CONF_LATITUDE: cv.latitude,
@@ -111,6 +117,7 @@ CORE_CONFIG_SCHEMA = vol.Schema({
     CONF_TIME_ZONE: cv.time_zone,
     vol.Required(CONF_CUSTOMIZE,
                  default=MappingProxyType({})): _valid_customize,
+    vol.Optional(CONF_PACKAGES, default={}): PACKAGES_CONFIG_SCHEMA,
 })
 
 
@@ -357,3 +364,91 @@ def async_process_ha_core_config(hass, config):
         _LOGGER.warning(
             'Incomplete core config. Auto detected %s',
             ', '.join('{}: {}'.format(key, val) for key, val in discovered))
+
+
+def _log_pkg_error(package, component, config, message):
+    """Log an error while merging."""
+    message = "Package {} setup failed. Component {} {}".format(
+        package, component, message)
+
+    pack_config = config[CONF_CORE][CONF_PACKAGES].get(package, config)
+    message += " (See {}:{}). ".format(
+        getattr(pack_config, '__config_file__', '?'),
+        getattr(pack_config, '__line__', '?'))
+
+    _LOGGER.error(message)
+
+
+def _identify_config_schema(module):
+    """Extract the schema and identify list or dict based."""
+    try:
+        schema = module.CONFIG_SCHEMA.schema[module.DOMAIN]
+    except (AttributeError, KeyError):
+        return (None, None)
+    t_schema = str(schema)
+    if (t_schema.startswith('<function ordered_dict') or
+            t_schema.startswith('<Schema({<function slug')):
+        return ('dict', schema)
+    if t_schema.startswith('All(<function ensure_list'):
+        return ('list', schema)
+    return '', schema
+
+
+def merge_packages_config(config, packages):
+    """Merge packages into the top-level config. Mutate config."""
+    # pylint: disable=too-many-nested-blocks
+    PACKAGES_CONFIG_SCHEMA(packages)
+    for pack_name, pack_conf in packages.items():
+        for comp_name, comp_conf in pack_conf.items():
+            component = get_component(comp_name)
+
+            if component is None:
+                _log_pkg_error(pack_name, comp_name, config, "does not exist")
+                continue
+
+            if hasattr(component, 'PLATFORM_SCHEMA'):
+                config[comp_name] = cv.ensure_list(config.get(comp_name))
+                config[comp_name].extend(cv.ensure_list(comp_conf))
+                continue
+
+            if hasattr(component, 'CONFIG_SCHEMA'):
+                merge_type, _ = _identify_config_schema(component)
+
+                if merge_type == 'list':
+                    config[comp_name] = cv.ensure_list(config.get(comp_name))
+                    config[comp_name].extend(cv.ensure_list(comp_conf))
+                    continue
+
+                if merge_type == 'dict':
+                    if not isinstance(comp_conf, dict):
+                        _log_pkg_error(
+                            pack_name, comp_name, config,
+                            "cannot be merged. Expected a dict.")
+                        continue
+
+                    if comp_name not in config:
+                        config[comp_name] = OrderedDict()
+
+                    if not isinstance(config[comp_name], dict):
+                        _log_pkg_error(
+                            pack_name, comp_name, config,
+                            "cannot be merged. Dict expected in main config.")
+                        continue
+
+                    for key, val in comp_conf.items():
+                        if key in config[comp_name]:
+                            _log_pkg_error(pack_name, comp_name, config,
+                                           "duplicate key '{}'".format(key))
+                            continue
+                        config[comp_name][key] = val
+                    continue
+
+            # The last merge type are sections that may occur only once
+            if comp_name in config:
+                _log_pkg_error(
+                    pack_name, comp_name, config, "may occur only once"
+                    " and it already exist in your main config")
+                continue
+            config[comp_name] = comp_conf
+
+    return config

homeassistant/const.py

@@ -109,6 +109,7 @@ CONF_MONITORED_VARIABLES = 'monitored_variables'
 CONF_NAME = 'name'
 CONF_OFFSET = 'offset'
 CONF_OPTIMISTIC = 'optimistic'
+CONF_PACKAGES = 'packages'
 CONF_PASSWORD = 'password'
 CONF_PATH = 'path'
 CONF_PAYLOAD = 'payload'

script/inspect_schemas.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""Inspect all component SCHEMAS."""
+import os
+import importlib
+import pkgutil
+
+from homeassistant.config import _identify_config_schema
+from homeassistant.scripts.check_config import color
+
+
+def explore_module(package):
+    """Explore the modules."""
+    module = importlib.import_module(package)
+    if not hasattr(module, '__path__'):
+        return []
+    for _, name, _ in pkgutil.iter_modules(module.__path__, package + '.'):
+        yield name
+
+
+def main():
+    """Main section of the script."""
+    if not os.path.isfile('requirements_all.txt'):
+        print('Run this from HA root dir')
+        return
+
+    msg = {}
+
+    def add_msg(key, item):
+        """Add a message."""
+        if key not in msg:
+            msg[key] = []
+        msg[key].append(item)
+
+    for package in explore_module('homeassistant.components'):
+        module = importlib.import_module(package)
+        module_name = getattr(module, 'DOMAIN', module.__name__)
+
+        if hasattr(module, 'PLATFORM_SCHEMA'):
+            if hasattr(module, 'CONFIG_SCHEMA'):
+                add_msg('WARNING', "Module {} contains PLATFORM and CONFIG "
+                        "schemas".format(module_name))
+            add_msg('PLATFORM SCHEMA', module_name)
+            continue
+
+        if not hasattr(module, 'CONFIG_SCHEMA'):
+            add_msg('NO SCHEMA', module_name)
+            continue
+
+        schema_type, schema = _identify_config_schema(module)
+
+        add_msg("CONFIG_SCHEMA " + schema_type, module_name + ' ' +
+                color('cyan', str(schema)[:60]))
+
+    for key in sorted(msg):
+        print("\n{}\n - {}".format(key, '\n - '.join(msg[key])))
+
+
+if __name__ == '__main__':
+    main()

tests/test_config.py

@@ -357,3 +357,128 @@ class TestConfig(unittest.TestCase):
         assert self.hass.config.location_name == blankConfig.location_name
         assert self.hass.config.units == blankConfig.units
         assert self.hass.config.time_zone == blankConfig.time_zone
+
+
+# pylint: disable=redefined-outer-name
+@pytest.fixture
+def merge_log_err(hass):
+    """Patch _merge_log_error from packages."""
+    with mock.patch('homeassistant.config._LOGGER.error') \
+            as logerr:
+        yield logerr
+
+
+def test_merge(merge_log_err):
+    """Test if we can merge packages."""
+    packages = {
+        'pack_dict': {'input_boolean': {'ib1': None}},
+        'pack_11': {'input_select': {'is1': None}},
+        'pack_list': {'light': {'platform': 'test'}},
+        'pack_list2': {'light': [{'platform': 'test'}]},
+    }
+    config = {
+        config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages},
+        'input_boolean': {'ib2': None},
+        'light': {'platform': 'test'}
+    }
+    config_util.merge_packages_config(config, packages)
+
+    assert merge_log_err.call_count == 0
+    assert len(config) == 4
+    assert len(config['input_boolean']) == 2
+    assert len(config['input_select']) == 1
+    assert len(config['light']) == 3
+
+
+def test_merge_new(merge_log_err):
+    """Test adding new components to outer scope."""
+    packages = {
+        'pack_1': {'light': [{'platform': 'one'}]},
+        'pack_11': {'input_select': {'ib1': None}},
+        'pack_2': {
+            'light': {'platform': 'one'},
+            'panel_custom': {'pan1': None},
+            'api': {}},
+    }
+    config = {
+        config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages},
+    }
+    config_util.merge_packages_config(config, packages)
+
+    assert merge_log_err.call_count == 0
+    assert 'api' in config
+    assert len(config) == 5
+    assert len(config['light']) == 2
+    assert len(config['panel_custom']) == 1
+
+
+def test_merge_type_mismatch(merge_log_err):
+    """Test if we have a type mismatch for packages."""
+    packages = {
+        'pack_1': {'input_boolean': [{'ib1': None}]},
+        'pack_11': {'input_select': {'ib1': None}},
+        'pack_2': {'light': {'ib1': None}},  # light gets merged - ensure_list
+    }
+    config = {
+        config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages},
+        'input_boolean': {'ib2': None},
+        'input_select': [{'ib2': None}],
+        'light': [{'platform': 'two'}]
+    }
+    config_util.merge_packages_config(config, packages)
+
+    assert merge_log_err.call_count == 2
+    assert len(config) == 4
+    assert len(config['input_boolean']) == 1
+    assert len(config['light']) == 2
+
+
+def test_merge_once_only(merge_log_err):
+    """Test if we have a merge for a comp that may occur only once."""
+    packages = {
+        'pack_1': {'homeassistant': {}},
+        'pack_2': {
+            'mqtt': {},
+            'api': {},  # No config schema
+        },
+    }
+    config = {
+        config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages},
+        'mqtt': {}, 'api': {}
+    }
+    config_util.merge_packages_config(config, packages)
+    assert merge_log_err.call_count == 3
+    assert len(config) == 3
+
+
+def test_merge_id_schema(hass):
+    """Test if we identify the config schemas correctly."""
+    types = {
+        'panel_custom': 'list',
+        'group': 'dict',
+        'script': 'dict',
+        'input_boolean': 'dict',
+        'shell_command': 'dict',
+        'qwikswitch': '',
+    }
+    for name, expected_type in types.items():
+        module = config_util.get_component(name)
+        typ, _ = config_util._identify_config_schema(module)
+        assert typ == expected_type, "{} expected {}, got {}".format(
+            name, expected_type, typ)
+
+
+def test_merge_duplicate_keys(merge_log_err):
+    """Test if keys in dicts are duplicates."""
+    packages = {
+        'pack_1': {'input_select': {'ib1': None}},
+    }
+    config = {
+        config_util.CONF_CORE: {config_util.CONF_PACKAGES: packages},
+        'input_select': {'ib1': None},
+    }
+    config_util.merge_packages_config(config, packages)
+
+    assert merge_log_err.call_count == 1
+    assert len(config) == 2
+    assert len(config['input_select']) == 1