Scrape move yaml config to integration key (#74325)

* Scrape take 2

* cleanup

* new entity name

* Fix name, add tests

* Use FlowResultType

* Add test abort

* issue

* hassfest

* Remove not needed test

* clean

* Remove config entry and implement datacoordinator

* fix codeowners

* fix codeowners

* codeowners reset

* Fix coordinator

* Remove test config_flow

* Fix tests

* hassfest

* reset config flow

* reset strings

* reset sensor

* next version

* Reconfig

* Adjust sensor

* cleanup sensor

* cleanup init

* Fix tests

* coverage

* Guard against empty sensor

* naming

* Remove coverage

* Review comments

* Remove print

* Move sensor check
This commit is contained in:
G Johansson 2022-10-30 13:02:11 +01:00 committed by GitHub
parent 06773efcbd
commit 662aee17a6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 498 additions and 114 deletions

View file

@ -1 +1,86 @@
"""The scrape component."""
from __future__ import annotations
from datetime import timedelta
import logging
import voluptuous as vol
from homeassistant.components.rest import RESOURCE_SCHEMA, create_rest_data_from_config
from homeassistant.components.sensor import DOMAIN as SENSOR_DOMAIN
from homeassistant.const import (
CONF_ATTRIBUTE,
CONF_SCAN_INTERVAL,
CONF_VALUE_TEMPLATE,
Platform,
)
from homeassistant.core import HomeAssistant
from homeassistant.exceptions import PlatformNotReady
from homeassistant.helpers import discovery
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.template_entity import TEMPLATE_SENSOR_BASE_SCHEMA
from homeassistant.helpers.typing import ConfigType
from .const import CONF_INDEX, CONF_SELECT, DEFAULT_SCAN_INTERVAL, DOMAIN
from .coordinator import ScrapeCoordinator
_LOGGER = logging.getLogger(__name__)
SENSOR_SCHEMA = vol.Schema(
{
**TEMPLATE_SENSOR_BASE_SCHEMA.schema,
vol.Optional(CONF_ATTRIBUTE): cv.string,
vol.Optional(CONF_INDEX, default=0): cv.positive_int,
vol.Required(CONF_SELECT): cv.string,
vol.Optional(CONF_VALUE_TEMPLATE): cv.template,
}
)
COMBINED_SCHEMA = vol.Schema(
{
vol.Optional(CONF_SCAN_INTERVAL): cv.time_period,
**RESOURCE_SCHEMA,
vol.Optional(SENSOR_DOMAIN): vol.All(
cv.ensure_list, [vol.Schema(SENSOR_SCHEMA)]
),
}
)
CONFIG_SCHEMA = vol.Schema(
{vol.Optional(DOMAIN): vol.All(cv.ensure_list, [COMBINED_SCHEMA])},
extra=vol.ALLOW_EXTRA,
)
async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool:
"""Set up Scrape from yaml config."""
if not (scrape_config := config.get(DOMAIN)):
return True
for resource_config in scrape_config:
if not (sensors := resource_config.get(SENSOR_DOMAIN)):
raise PlatformNotReady("No sensors configured")
rest = create_rest_data_from_config(hass, resource_config)
coordinator = ScrapeCoordinator(
hass,
rest,
timedelta(
seconds=resource_config.get(CONF_SCAN_INTERVAL, DEFAULT_SCAN_INTERVAL)
),
)
await coordinator.async_refresh()
if coordinator.data is None:
raise PlatformNotReady
for sensor_config in sensors:
discovery.load_platform(
hass,
Platform.SENSOR,
DOMAIN,
{"coordinator": coordinator, "config": sensor_config},
config,
)
return True

View file

@ -6,6 +6,7 @@ from homeassistant.const import Platform
DOMAIN = "scrape"
DEFAULT_NAME = "Web scrape"
DEFAULT_VERIFY_SSL = True
DEFAULT_SCAN_INTERVAL = 60 * 10
PLATFORMS = [Platform.SENSOR]

View file

@ -34,6 +34,7 @@ from homeassistant.core import HomeAssistant, callback
from homeassistant.exceptions import PlatformNotReady
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.issue_registry import IssueSeverity, async_create_issue
from homeassistant.helpers.template import Template
from homeassistant.helpers.template_entity import (
TEMPLATE_SENSOR_BASE_SCHEMA,
@ -42,7 +43,7 @@ from homeassistant.helpers.template_entity import (
from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
from homeassistant.helpers.update_coordinator import CoordinatorEntity
from .const import CONF_INDEX, CONF_SELECT, DEFAULT_NAME, DEFAULT_VERIFY_SSL
from .const import CONF_INDEX, CONF_SELECT, DEFAULT_NAME, DEFAULT_VERIFY_SSL, DOMAIN
from .coordinator import ScrapeCoordinator
_LOGGER = logging.getLogger(__name__)
@ -82,25 +83,40 @@ async def async_setup_platform(
discovery_info: DiscoveryInfoType | None = None,
) -> None:
"""Set up the Web scrape sensor."""
resource_config = vol.Schema(RESOURCE_SCHEMA, extra=vol.REMOVE_EXTRA)(config)
rest = create_rest_data_from_config(hass, resource_config)
if discovery_info is None:
async_create_issue(
hass,
DOMAIN,
"moved_yaml",
breaks_in_ha_version="2022.12.0",
is_fixable=False,
severity=IssueSeverity.WARNING,
translation_key="moved_yaml",
)
resource_config = vol.Schema(RESOURCE_SCHEMA, extra=vol.REMOVE_EXTRA)(config)
rest = create_rest_data_from_config(hass, resource_config)
coordinator = ScrapeCoordinator(hass, rest, SCAN_INTERVAL)
await coordinator.async_refresh()
if coordinator.data is None:
raise PlatformNotReady
coordinator = ScrapeCoordinator(hass, rest, SCAN_INTERVAL)
await coordinator.async_refresh()
if coordinator.data is None:
raise PlatformNotReady
sensor_config = vol.Schema(
TEMPLATE_SENSOR_BASE_SCHEMA.schema, extra=vol.REMOVE_EXTRA
)(config)
sensor_config = config
template_config = vol.Schema(
TEMPLATE_SENSOR_BASE_SCHEMA.schema, extra=vol.REMOVE_EXTRA
)(sensor_config)
name: str = config[CONF_NAME]
unique_id: str | None = config.get(CONF_UNIQUE_ID)
else:
coordinator = discovery_info["coordinator"]
sensor_config = discovery_info["config"]
template_config = sensor_config
select: str | None = config.get(CONF_SELECT)
attr: str | None = config.get(CONF_ATTRIBUTE)
index: int = config[CONF_INDEX]
value_template: Template | None = config.get(CONF_VALUE_TEMPLATE)
name: str = template_config[CONF_NAME]
unique_id: str | None = template_config.get(CONF_UNIQUE_ID)
select: str | None = sensor_config.get(CONF_SELECT)
attr: str | None = sensor_config.get(CONF_ATTRIBUTE)
index: int = sensor_config[CONF_INDEX]
value_template: Template | None = sensor_config.get(CONF_VALUE_TEMPLATE)
if value_template is not None:
value_template.hass = hass
@ -110,7 +126,7 @@ async def async_setup_platform(
ScrapeSensor(
hass,
coordinator,
sensor_config,
template_config,
name,
unique_id,
select,

View file

@ -69,5 +69,11 @@
}
}
}
},
"issues": {
"moved_yaml": {
"title": "The Scrape YAML configuration has been moved",
"description": "Configuring Scrape using YAML has been moved to integration key.\n\nYour existing YAML configuration will be working for 2 more versions.\n\nMigrate your YAML configuration to the integration key according to the documentation."
}
}
}

View file

@ -36,6 +36,12 @@
}
}
},
"issues": {
"moved_yaml": {
"description": "Configuring Scrape using YAML has been moved to integration key.\n\nYour existing YAML configuration will be working for 2 more versions.\n\nMigrate your YAML configuration to the integration key according to the documentation.",
"title": "The Scrape YAML configuration has been moved"
}
},
"options": {
"step": {
"init": {

View file

@ -4,6 +4,31 @@ from __future__ import annotations
from typing import Any
def return_integration_config(
*,
authentication=None,
username=None,
password=None,
headers=None,
sensors=None,
) -> dict[str, dict[str, Any]]:
"""Return config."""
config = {
"resource": "https://www.home-assistant.io",
"verify_ssl": True,
"sensor": sensors,
}
if authentication:
config["authentication"] = authentication
if username:
config["username"] = username
config["password"] = password
if headers:
config["headers"] = headers
return config
def return_config(
select,
name,
@ -19,6 +44,7 @@ def return_config(
password=None,
headers=None,
unique_id=None,
remove_platform=False,
) -> dict[str, dict[str, Any]]:
"""Return config."""
config = {
@ -26,7 +52,11 @@ def return_config(
"resource": "https://www.home-assistant.io",
"select": select,
"name": name,
"index": 0,
"verify_ssl": True,
}
if remove_platform:
config.pop("platform")
if attribute:
config["attribute"] = attribute
if index:
@ -41,6 +71,7 @@ def return_config(
config["state_class"] = state_class
if authentication:
config["authentication"] = authentication
if username:
config["username"] = username
config["password"] = password
if headers:

View file

@ -0,0 +1,67 @@
"""Test Scrape component setup process."""
from __future__ import annotations
from unittest.mock import patch
from homeassistant.components.scrape.const import DOMAIN
from homeassistant.core import HomeAssistant
from homeassistant.helpers import entity_registry as er
from homeassistant.setup import async_setup_component
from . import MockRestData, return_integration_config
async def test_setup_config(hass: HomeAssistant) -> None:
"""Test setup from yaml."""
config = {
DOMAIN: [
return_integration_config(
sensors=[{"select": ".current-version h1", "name": "HA version"}]
)
]
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
) as mock_setup:
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state.state == "Current Version: 2021.12.10"
assert len(mock_setup.mock_calls) == 1
async def test_setup_no_data_fails(hass: HomeAssistant) -> None:
"""Test setup entry no data fails."""
config = {
DOMAIN: [
return_integration_config(
sensors=[{"select": ".current-version h1", "name": "HA version"}]
),
]
}
with patch(
"homeassistant.components.scrape.coordinator.RestData",
return_value=MockRestData("test_scrape_sensor_no_data"),
):
assert not await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state is None
async def test_setup_config_no_configuration(hass: HomeAssistant) -> None:
"""Test setup from yaml missing configuration options."""
config = {DOMAIN: None}
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
entities = er.async_get(hass)
assert entities.entities == {}

View file

@ -7,6 +7,7 @@ from unittest.mock import patch
from homeassistant.components.scrape.sensor import SCAN_INTERVAL
from homeassistant.components.sensor import (
CONF_STATE_CLASS,
DOMAIN as SENSOR_DOMAIN,
SensorDeviceClass,
SensorStateClass,
)
@ -21,7 +22,7 @@ from homeassistant.core import HomeAssistant
from homeassistant.helpers import entity_registry as er
from homeassistant.setup import async_setup_component
from . import MockRestData, return_config
from . import MockRestData, return_config, return_integration_config
from tests.common import async_fire_time_changed
@ -30,102 +31,30 @@ DOMAIN = "scrape"
async def test_scrape_sensor(hass: HomeAssistant) -> None:
"""Test Scrape sensor minimal."""
config = {"sensor": return_config(select=".current-version h1", name="HA version")}
config = {
DOMAIN: [
return_integration_config(
sensors=[{"select": ".current-version h1", "name": "HA version"}]
)
]
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state.state == "Current Version: 2021.12.10"
async def test_scrape_sensor_value_template(hass: HomeAssistant) -> None:
"""Test Scrape sensor with value template."""
async def test_scrape_sensor_platform_yaml(hass: HomeAssistant) -> None:
"""Test Scrape sensor load from sensor platform."""
config = {
"sensor": return_config(
select=".current-version h1",
name="HA version",
template="{{ value.split(':')[1] }}",
)
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state.state == "2021.12.10"
async def test_scrape_uom_and_classes(hass: HomeAssistant) -> None:
"""Test Scrape sensor for unit of measurement, device class and state class."""
config = {
"sensor": return_config(
select=".current-temp h3",
name="Current Temp",
template="{{ value.split(':')[1] }}",
uom="°C",
device_class="temperature",
state_class="measurement",
)
}
mocker = MockRestData("test_scrape_uom_and_classes")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.current_temp")
assert state.state == "22.1"
assert state.attributes[CONF_UNIT_OF_MEASUREMENT] == TEMP_CELSIUS
assert state.attributes[CONF_DEVICE_CLASS] == SensorDeviceClass.TEMPERATURE
assert state.attributes[CONF_STATE_CLASS] == SensorStateClass.MEASUREMENT
async def test_scrape_unique_id(hass: HomeAssistant) -> None:
"""Test Scrape sensor for unique id."""
config = {
"sensor": return_config(
select=".current-temp h3",
name="Current Temp",
template="{{ value.split(':')[1] }}",
unique_id="very_unique_id",
)
}
mocker = MockRestData("test_scrape_uom_and_classes")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.current_temp")
assert state.state == "22.1"
registry = er.async_get(hass)
entry = registry.async_get("sensor.current_temp")
assert entry
assert entry.unique_id == "very_unique_id"
async def test_scrape_sensor_authentication(hass: HomeAssistant) -> None:
"""Test Scrape sensor with authentication."""
config = {
"sensor": [
SENSOR_DOMAIN: [
return_config(
select=".return",
name="Auth page",
@ -147,7 +76,170 @@ async def test_scrape_sensor_authentication(hass: HomeAssistant) -> None:
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
assert await async_setup_component(hass, SENSOR_DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.auth_page")
assert state.state == "secret text"
state2 = hass.states.get("sensor.auth_page2")
assert state2.state == "secret text"
async def test_scrape_sensor_platform_yaml_no_data(hass: HomeAssistant, caplog) -> None:
"""Test Scrape sensor load from sensor platform fetching no data."""
config = {
SENSOR_DOMAIN: [
return_config(
select=".return",
name="Auth page",
username="user@secret.com",
password="12345678",
authentication="digest",
),
]
}
mocker = MockRestData("test_scrape_sensor_no_data")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, SENSOR_DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.auth_page")
assert not state
assert "Platform scrape not ready yet: None; Retrying in background" in caplog.text
async def test_scrape_sensor_value_template(hass: HomeAssistant) -> None:
"""Test Scrape sensor with value template."""
config = {
DOMAIN: [
return_integration_config(
sensors=[
{
"select": ".current-version h1",
"name": "HA version",
"value_template": "{{ value.split(':')[1] }}",
}
]
)
]
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state.state == "2021.12.10"
async def test_scrape_uom_and_classes(hass: HomeAssistant) -> None:
"""Test Scrape sensor for unit of measurement, device class and state class."""
config = {
DOMAIN: [
return_integration_config(
sensors=[
{
"select": ".current-temp h3",
"name": "Current Temp",
"value_template": "{{ value.split(':')[1] }}",
"unit_of_measurement": "°C",
"device_class": "temperature",
"state_class": "measurement",
}
]
)
]
}
mocker = MockRestData("test_scrape_uom_and_classes")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.current_temp")
assert state.state == "22.1"
assert state.attributes[CONF_UNIT_OF_MEASUREMENT] == TEMP_CELSIUS
assert state.attributes[CONF_DEVICE_CLASS] == SensorDeviceClass.TEMPERATURE
assert state.attributes[CONF_STATE_CLASS] == SensorStateClass.MEASUREMENT
async def test_scrape_unique_id(hass: HomeAssistant) -> None:
"""Test Scrape sensor for unique id."""
config = {
DOMAIN: return_integration_config(
sensors=[
{
"select": ".current-temp h3",
"name": "Current Temp",
"value_template": "{{ value.split(':')[1] }}",
"unique_id": "very_unique_id",
}
]
)
}
mocker = MockRestData("test_scrape_uom_and_classes")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.current_temp")
assert state.state == "22.1"
registry = er.async_get(hass)
entry = registry.async_get("sensor.current_temp")
assert entry
assert entry.unique_id == "very_unique_id"
async def test_scrape_sensor_authentication(hass: HomeAssistant) -> None:
"""Test Scrape sensor with authentication."""
config = {
DOMAIN: [
return_integration_config(
authentication="digest",
username="user@secret.com",
password="12345678",
sensors=[
{
"select": ".return",
"name": "Auth page",
},
],
),
return_integration_config(
username="user@secret.com",
password="12345678",
sensors=[
{
"select": ".return",
"name": "Auth page2",
},
],
),
]
}
mocker = MockRestData("test_scrape_sensor_authentication")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.auth_page")
@ -158,14 +250,18 @@ async def test_scrape_sensor_authentication(hass: HomeAssistant) -> None:
async def test_scrape_sensor_no_data(hass: HomeAssistant) -> None:
"""Test Scrape sensor fails on no data."""
config = {"sensor": return_config(select=".current-version h1", name="HA version")}
config = {
DOMAIN: return_integration_config(
sensors=[{"select": ".current-version h1", "name": "HA version"}]
)
}
mocker = MockRestData("test_scrape_sensor_no_data")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
assert not await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
@ -174,14 +270,20 @@ async def test_scrape_sensor_no_data(hass: HomeAssistant) -> None:
async def test_scrape_sensor_no_data_refresh(hass: HomeAssistant) -> None:
"""Test Scrape sensor no data on refresh."""
config = {"sensor": return_config(select=".current-version h1", name="HA version")}
config = {
DOMAIN: [
return_integration_config(
sensors=[{"select": ".current-version h1", "name": "HA version"}]
)
]
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
@ -200,9 +302,18 @@ async def test_scrape_sensor_no_data_refresh(hass: HomeAssistant) -> None:
async def test_scrape_sensor_attribute_and_tag(hass: HomeAssistant) -> None:
"""Test Scrape sensor with attribute and tag."""
config = {
"sensor": [
return_config(select="div", name="HA class", index=1, attribute="class"),
return_config(select="template", name="HA template"),
DOMAIN: [
return_integration_config(
sensors=[
{
"index": 1,
"select": "div",
"name": "HA class",
"attribute": "class",
},
{"select": "template", "name": "HA template"},
],
),
]
}
@ -211,7 +322,7 @@ async def test_scrape_sensor_attribute_and_tag(hass: HomeAssistant) -> None:
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_class")
@ -223,9 +334,22 @@ async def test_scrape_sensor_attribute_and_tag(hass: HomeAssistant) -> None:
async def test_scrape_sensor_errors(hass: HomeAssistant) -> None:
"""Test Scrape sensor handle errors."""
config = {
"sensor": [
return_config(select="div", name="HA class", index=5, attribute="class"),
return_config(select="div", name="HA class2", attribute="classes"),
DOMAIN: [
return_integration_config(
sensors=[
{
"index": 5,
"select": "div",
"name": "HA class",
"attribute": "class",
},
{
"select": "div",
"name": "HA class2",
"attribute": "classes",
},
],
),
]
}
@ -234,10 +358,58 @@ async def test_scrape_sensor_errors(hass: HomeAssistant) -> None:
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_class")
assert state.state == STATE_UNKNOWN
state2 = hass.states.get("sensor.ha_class2")
assert state2.state == STATE_UNKNOWN
async def test_scrape_sensor_unique_id(hass: HomeAssistant) -> None:
"""Test Scrape sensor with unique_id."""
config = {
DOMAIN: [
return_integration_config(
sensors=[
{
"select": ".current-version h1",
"name": "HA version",
"unique_id": "ha_version_unique_id",
}
]
)
]
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state.state == "Current Version: 2021.12.10"
entity_reg = er.async_get(hass)
entity = entity_reg.async_get("sensor.ha_version")
assert entity.unique_id == "ha_version_unique_id"
async def test_scrape_sensor_not_configured_sensor(hass: HomeAssistant, caplog) -> None:
"""Test Scrape sensor with missing configured sensors."""
config = {DOMAIN: [return_integration_config(sensors=None)]}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.rest.RestData",
return_value=mocker,
):
assert not await async_setup_component(hass, DOMAIN, config)
await hass.async_block_till_done()
assert "No sensors configured" in caplog.text