Add tests for configuration validation errors (#103848)

* Add tests for configuration validation errors

* Use absolute path for hass.config.config_dir

* Apply suggestions from code review

---------

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

tests/fixtures/core/config/basic/configuration.yaml

@@ -0,0 +1,21 @@
+iot_domain:
+  # This is correct and should not generate errors
+  - platform: non_adr_0007
+    option1: abc
+  # This violates the non_adr_0007.iot_domain platform schema
+  - platform: non_adr_0007
+    option1: 123
+  # This violates the iot_domain platform schema
+  - paltfrom: non_adr_0007
+
+# This is correct and should not generate errors
+adr_0007_1:
+  host: blah.com
+
+# Host is missing
+adr_0007_2:
+
+# Port is wrong type
+adr_0007_3:
+  host: blah.com
+  port: foo

tests/fixtures/core/config/basic_include/configuration.yaml

@@ -0,0 +1,4 @@
+iot_domain: !include integrations/iot_domain.yaml
+adr_0007_1: !include integrations/adr_0007_1.yaml
+adr_0007_2: !include integrations/adr_0007_2.yaml
+adr_0007_3: !include integrations/adr_0007_3.yaml

tests/fixtures/core/config/basic_include/integrations/adr_0007_1.yaml

@@ -0,0 +1,2 @@
+# This is correct and should not generate errors
+host: blah.com

tests/fixtures/core/config/basic_include/integrations/adr_0007_2.yaml

@@ -0,0 +1 @@
+# Host is missing

tests/fixtures/core/config/basic_include/integrations/adr_0007_3.yaml

@@ -0,0 +1,3 @@
+# Port is wrong type
+host: blah.com
+port: foo

tests/fixtures/core/config/basic_include/integrations/iot_domain.yaml

@@ -0,0 +1,8 @@
+# This is correct and should not generate errors
+- platform: non_adr_0007
+  option1: abc
+# This violates the non_adr_0007.iot_domain platform schema
+- platform: non_adr_0007
+  option1: 123
+# This violates the iot_domain platform schema
+- paltfrom: non_adr_0007

tests/fixtures/core/config/include_dir_list/configuration.yaml

@@ -0,0 +1 @@
+iot_domain: !include_dir_list iot_domain

tests/fixtures/core/config/include_dir_list/iot_domain/iot_domain_1.yaml

@@ -0,0 +1,3 @@
+# This is correct and should not generate errors
+platform: non_adr_0007
+option1: abc

tests/fixtures/core/config/include_dir_list/iot_domain/iot_domain_2.yaml

@@ -0,0 +1,3 @@
+# This violates the non_adr_0007.iot_domain platform schema
+platform: non_adr_0007
+option1: 123

tests/fixtures/core/config/include_dir_list/iot_domain/iot_domain_3.yaml

@@ -0,0 +1,2 @@
+# This violates the iot_domain platform schema
+paltfrom: non_adr_0007

tests/fixtures/core/config/include_dir_merge_list/configuration.yaml

@@ -0,0 +1 @@
+iot_domain: !include_dir_merge_list iot_domain

tests/fixtures/core/config/include_dir_merge_list/iot_domain/iot_domain_1.yaml

@@ -0,0 +1,3 @@
+# This is correct and should not generate errors
+- platform: non_adr_0007
+  option1: abc

tests/fixtures/core/config/include_dir_merge_list/iot_domain/iot_domain_2.yaml

@@ -0,0 +1,5 @@
+# This violates the non_adr_0007.iot_domain platform schema
+- platform: non_adr_0007
+  option1: 123
+  # This violates the iot_domain platform schema
+- paltfrom: non_adr_0007

tests/fixtures/core/config/packages/configuration.yaml

@@ -0,0 +1,28 @@
+homeassistant:
+  packages:
+    pack_1:
+      iot_domain:
+        # This is correct and should not generate errors
+        - platform: non_adr_0007
+          option1: abc
+    pack_2:
+      iot_domain:
+        # This violates the non_adr_0007.iot_domain platform schema
+        - platform: non_adr_0007
+          option1: 123
+    pack_3:
+      iot_domain:
+        # This violates the iot_domain platform schema
+        - paltfrom: non_adr_0007
+    pack_4:
+      # This is correct and should not generate errors
+      adr_0007_1:
+        host: blah.com
+    pack_5:
+      # Host is missing
+      adr_0007_2:
+    pack_6:
+      # Port is wrong type
+      adr_0007_3:
+        host: blah.com
+        port: foo

tests/fixtures/core/config/packages_include_dir_named/configuration.yaml

@@ -0,0 +1,3 @@
+homeassistant:
+  # Load packages
+  packages: !include_dir_named integrations

tests/fixtures/core/config/packages_include_dir_named/integrations/adr_0007_1.yaml

@@ -0,0 +1,3 @@
+# This is correct and should not generate errors
+adr_0007_1:
+  host: blah.com

tests/fixtures/core/config/packages_include_dir_named/integrations/adr_0007_2.yaml

@@ -0,0 +1,2 @@
+# Host is missing
+adr_0007_2:

tests/fixtures/core/config/packages_include_dir_named/integrations/adr_0007_3.yaml

@@ -0,0 +1,4 @@
+# Port is wrong type
+adr_0007_3:
+  host: blah.com
+  port: foo

tests/fixtures/core/config/packages_include_dir_named/integrations/iot_domain.yaml

@@ -0,0 +1,9 @@
+iot_domain:
+  # This is correct and should not generate errors
+  - platform: non_adr_0007
+    option1: abc
+  # This violates the non_adr_0007.iot_domain platform schema
+  - platform: non_adr_0007
+    option1: 123
+  # This violates the iot_domain platform schema
+  - paltfrom: non_adr_0007

tests/snapshots/test_config.ambr

@@ -0,0 +1,45 @@
+# serializer version: 1
+# name: test_component_config_validation_error[basic]
+  list([
+    "Invalid config for [iot_domain.non_adr_0007]: expected str for dictionary value @ data['option1']. Got 123. (See <BASE_PATH>/fixtures/core/config/basic/configuration.yaml, line 6). ",
+    "Invalid config for [iot_domain]: required key not provided @ data['platform']. Got None. (See <BASE_PATH>/fixtures/core/config/basic/configuration.yaml, line 9). ",
+    "Invalid config for [adr_0007_2]: required key not provided @ data['adr_0007_2']['host']. Got None. (See ?, line ?). ",
+    "Invalid config for [adr_0007_3]: expected int for dictionary value @ data['adr_0007_3']['port']. Got 'foo'. (See <BASE_PATH>/fixtures/core/config/basic/configuration.yaml, line 20). ",
+  ])
+# ---
+# name: test_component_config_validation_error[basic_include]
+  list([
+    "Invalid config for [iot_domain.non_adr_0007]: expected str for dictionary value @ data['option1']. Got 123. (See <BASE_PATH>/fixtures/core/config/basic_include/integrations/iot_domain.yaml, line 5). ",
+    "Invalid config for [iot_domain]: required key not provided @ data['platform']. Got None. (See <BASE_PATH>/fixtures/core/config/basic_include/integrations/iot_domain.yaml, line 8). ",
+    "Invalid config for [adr_0007_2]: required key not provided @ data['adr_0007_2']['host']. Got None. (See ?, line ?). ",
+    "Invalid config for [adr_0007_3]: expected int for dictionary value @ data['adr_0007_3']['port']. Got 'foo'. (See <BASE_PATH>/fixtures/core/config/basic_include/configuration.yaml, line 4). ",
+  ])
+# ---
+# name: test_component_config_validation_error[include_dir_list]
+  list([
+    "Invalid config for [iot_domain.non_adr_0007]: expected str for dictionary value @ data['option1']. Got 123. (See <BASE_PATH>/fixtures/core/config/include_dir_list/iot_domain/iot_domain_2.yaml, line 2). ",
+    "Invalid config for [iot_domain]: required key not provided @ data['platform']. Got None. (See <BASE_PATH>/fixtures/core/config/include_dir_list/iot_domain/iot_domain_3.yaml, line 2). ",
+  ])
+# ---
+# name: test_component_config_validation_error[include_dir_merge_list]
+  list([
+    "Invalid config for [iot_domain.non_adr_0007]: expected str for dictionary value @ data['option1']. Got 123. (See <BASE_PATH>/fixtures/core/config/include_dir_merge_list/iot_domain/iot_domain_2.yaml, line 2). ",
+    "Invalid config for [iot_domain]: required key not provided @ data['platform']. Got None. (See <BASE_PATH>/fixtures/core/config/include_dir_merge_list/iot_domain/iot_domain_2.yaml, line 5). ",
+  ])
+# ---
+# name: test_component_config_validation_error[packages]
+  list([
+    "Invalid config for [iot_domain.non_adr_0007]: expected str for dictionary value @ data['option1']. Got 123. (See <BASE_PATH>/fixtures/core/config/packages/configuration.yaml, line 11). ",
+    "Invalid config for [iot_domain]: required key not provided @ data['platform']. Got None. (See <BASE_PATH>/fixtures/core/config/packages/configuration.yaml, line 16). ",
+    "Invalid config for [adr_0007_2]: required key not provided @ data['adr_0007_2']['host']. Got None. (See ?, line ?). ",
+    "Invalid config for [adr_0007_3]: expected int for dictionary value @ data['adr_0007_3']['port']. Got 'foo'. (See ?, line ?). ",
+  ])
+# ---
+# name: test_component_config_validation_error[packages_include_dir_named]
+  list([
+    "Invalid config for [iot_domain.non_adr_0007]: expected str for dictionary value @ data['option1']. Got 123. (See <BASE_PATH>/fixtures/core/config/packages_include_dir_named/integrations/iot_domain.yaml, line 6). ",
+    "Invalid config for [iot_domain]: required key not provided @ data['platform']. Got None. (See <BASE_PATH>/fixtures/core/config/packages_include_dir_named/integrations/iot_domain.yaml, line 9). ",
+    "Invalid config for [adr_0007_2]: required key not provided @ data['adr_0007_2']['host']. Got None. (See ?, line ?). ",
+    "Invalid config for [adr_0007_3]: expected int for dictionary value @ data['adr_0007_3']['port']. Got 'foo'. (See ?, line ?). ",
+  ])
+# ---

tests/test_config.py

@@ -2,12 +2,14 @@
 from collections import OrderedDict
 import contextlib
 import copy
+import logging
 import os
 from typing import Any
 from unittest import mock
 from unittest.mock import AsyncMock, Mock, patch
 
 import pytest
+from syrupy.assertion import SnapshotAssertion
 import voluptuous as vol
 from voluptuous import Invalid, MultipleInvalid
 import yaml
@@ -40,7 +42,14 @@ from homeassistant.util.unit_system import (
 )
 from homeassistant.util.yaml import SECRET_YAML
 
-from .common import MockUser, get_test_config_dir
+from .common import (
+    MockModule,
+    MockPlatform,
+    MockUser,
+    get_test_config_dir,
+    mock_integration,
+    mock_platform,
+)
 
 CONFIG_DIR = get_test_config_dir()
 YAML_PATH = os.path.join(CONFIG_DIR, config_util.YAML_CONFIG_FILE)
@@ -1399,3 +1408,83 @@ async def test_safe_mode(hass: HomeAssistant) -> None:
     await config_util.async_enable_safe_mode(hass)
     assert config_util.safe_mode_enabled(hass.config.config_dir) is True
     assert config_util.safe_mode_enabled(hass.config.config_dir) is False
+
+
+@pytest.mark.parametrize(
+    "config_dir",
+    [
+        "basic",
+        "basic_include",
+        "include_dir_list",
+        "include_dir_merge_list",
+        "packages",
+        "packages_include_dir_named",
+    ],
+)
+async def test_component_config_validation_error(
+    hass: HomeAssistant,
+    caplog: pytest.LogCaptureFixture,
+    config_dir: str,
+    snapshot: SnapshotAssertion,
+) -> None:
+    """Test schema error in component."""
+    comp_platform_schema = cv.PLATFORM_SCHEMA.extend({vol.Remove("old"): str})
+    comp_platform_schema_base = comp_platform_schema.extend({}, extra=vol.ALLOW_EXTRA)
+
+    # Mock an integration which provides an IoT domain
+    mock_integration(
+        hass,
+        MockModule(
+            "iot_domain",
+            platform_schema_base=comp_platform_schema_base,
+            platform_schema=comp_platform_schema,
+        ),
+    )
+
+    # Mock a non-ADR-0007 compliant integration which allows setting up
+    # iot_domain entities under the iot_domain's configuration key
+    test_platform_schema = comp_platform_schema.extend({"option1": str})
+    mock_platform(
+        hass,
+        "non_adr_0007.iot_domain",
+        MockPlatform(platform_schema=test_platform_schema),
+    )
+
+    # Mock an ADR-0007 compliant integration
+    for domain in ["adr_0007_1", "adr_0007_2", "adr_0007_3"]:
+        adr_0007_config_schema = vol.Schema(
+            {
+                domain: vol.Schema(
+                    {
+                        vol.Required("host"): str,
+                        vol.Required("port", default=8080): int,
+                    }
+                )
+            },
+            extra=vol.ALLOW_EXTRA,
+        )
+        mock_integration(
+            hass,
+            MockModule(domain, config_schema=adr_0007_config_schema),
+        )
+
+    base_path = os.path.dirname(__file__)
+    hass.config.config_dir = os.path.join(
+        base_path, "fixtures", "core", "config", config_dir
+    )
+    config = await config_util.async_hass_config_yaml(hass)
+
+    for domain in ["iot_domain", "adr_0007_1", "adr_0007_2", "adr_0007_3"]:
+        integration = await async_get_integration(hass, domain)
+        await config_util.async_process_component_config(
+            hass,
+            config,
+            integration=integration,
+        )
+
+    error_records = [
+        record.message.replace(base_path, "<BASE_PATH>")
+        for record in caplog.get_records("call")
+        if record.levelno == logging.ERROR
+    ]
+    assert error_records == snapshot