From 6d20e68e6dbb51c1023d4ee85bf706fc180b4acd Mon Sep 17 00:00:00 2001 From: G Johansson Date: Sat, 12 Feb 2022 15:28:54 +0100 Subject: [PATCH] Code quality scrape (#65441) --- .coveragerc | 1 - CODEOWNERS | 1 + homeassistant/components/scrape/sensor.py | 89 +++++----- requirements_test_all.txt | 3 + tests/components/scrape/__init__.py | 83 +++++++++ tests/components/scrape/test_sensor.py | 206 ++++++++++++++++++++++ 6 files changed, 334 insertions(+), 49 deletions(-) create mode 100644 tests/components/scrape/__init__.py create mode 100644 tests/components/scrape/test_sensor.py diff --git a/.coveragerc b/.coveragerc index a4819a124c9..5fa34f27656 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1009,7 +1009,6 @@ omit = homeassistant/components/samsungtv/diagnostics.py homeassistant/components/satel_integra/* homeassistant/components/schluter/* - homeassistant/components/scrape/sensor.py homeassistant/components/screenlogic/__init__.py homeassistant/components/screenlogic/binary_sensor.py homeassistant/components/screenlogic/climate.py diff --git a/CODEOWNERS b/CODEOWNERS index b7b688a0e73..ceefad2bad4 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -808,6 +808,7 @@ homeassistant/components/scene/* @home-assistant/core tests/components/scene/* @home-assistant/core homeassistant/components/schluter/* @prairieapps homeassistant/components/scrape/* @fabaff +tests/components/scrape/* @fabaff homeassistant/components/screenlogic/* @dieselrabbit @bdraco tests/components/screenlogic/* @dieselrabbit @bdraco homeassistant/components/script/* @home-assistant/core diff --git a/homeassistant/components/scrape/sensor.py b/homeassistant/components/scrape/sensor.py index c0523e4cbe2..8f2a672ef06 100644 --- a/homeassistant/components/scrape/sensor.py +++ b/homeassistant/components/scrape/sensor.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +from typing import Any from bs4 import BeautifulSoup import httpx @@ -11,7 +12,7 @@ from homeassistant.components.rest.data import RestData from homeassistant.components.sensor import ( CONF_STATE_CLASS, DEVICE_CLASSES_SCHEMA, - PLATFORM_SCHEMA, + PLATFORM_SCHEMA as PARENT_PLATFORM_SCHEMA, STATE_CLASSES_SCHEMA, SensorEntity, ) @@ -33,6 +34,7 @@ from homeassistant.core import HomeAssistant from homeassistant.exceptions import PlatformNotReady import homeassistant.helpers.config_validation as cv from homeassistant.helpers.entity_platform import AddEntitiesCallback +from homeassistant.helpers.template import Template from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType _LOGGER = logging.getLogger(__name__) @@ -44,7 +46,7 @@ CONF_INDEX = "index" DEFAULT_NAME = "Web scrape" DEFAULT_VERIFY_SSL = True -PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( +PLATFORM_SCHEMA = PARENT_PLATFORM_SCHEMA.extend( { vol.Required(CONF_RESOURCE): cv.string, vol.Required(CONF_SELECT): cv.string, @@ -73,32 +75,32 @@ async def async_setup_platform( discovery_info: DiscoveryInfoType | None = None, ) -> None: """Set up the Web scrape sensor.""" - name = config.get(CONF_NAME) - resource = config.get(CONF_RESOURCE) - method = "GET" - payload = None - headers = config.get(CONF_HEADERS) - verify_ssl = config.get(CONF_VERIFY_SSL) - select = config.get(CONF_SELECT) - attr = config.get(CONF_ATTR) - index = config.get(CONF_INDEX) - unit = config.get(CONF_UNIT_OF_MEASUREMENT) - device_class = config.get(CONF_DEVICE_CLASS) - state_class = config.get(CONF_STATE_CLASS) - username = config.get(CONF_USERNAME) - password = config.get(CONF_PASSWORD) + name: str = config[CONF_NAME] + resource: str = config[CONF_RESOURCE] + method: str = "GET" + payload: str | None = None + headers: str | None = config.get(CONF_HEADERS) + verify_ssl: bool = config[CONF_VERIFY_SSL] + select: str | None = config.get(CONF_SELECT) + attr: str | None = config.get(CONF_ATTR) + index: int = config[CONF_INDEX] + unit: str | None = config.get(CONF_UNIT_OF_MEASUREMENT) + device_class: str | None = config.get(CONF_DEVICE_CLASS) + state_class: str | None = config.get(CONF_STATE_CLASS) + username: str | None = config.get(CONF_USERNAME) + password: str | None = config.get(CONF_PASSWORD) + value_template: Template | None = config.get(CONF_VALUE_TEMPLATE) - if (value_template := config.get(CONF_VALUE_TEMPLATE)) is not None: + if value_template is not None: value_template.hass = hass - auth: httpx.DigestAuth | tuple[str, str] | None + auth: httpx.DigestAuth | tuple[str, str] | None = None if username and password: if config.get(CONF_AUTHENTICATION) == HTTP_DIGEST_AUTHENTICATION: auth = httpx.DigestAuth(username, password) else: auth = (username, password) - else: - auth = None + rest = RestData(hass, method, resource, auth, headers, None, payload, verify_ssl) await rest.async_update() @@ -128,19 +130,19 @@ class ScrapeSensor(SensorEntity): def __init__( self, - rest, - name, - select, - attr, - index, - value_template, - unit, - device_class, - state_class, - ): + rest: RestData, + name: str, + select: str | None, + attr: str | None, + index: int, + value_template: Template | None, + unit: str | None, + device_class: str | None, + state_class: str | None, + ) -> None: """Initialize a web scrape sensor.""" self.rest = rest - self._state = None + self._attr_native_value = None self._select = select self._attr = attr self._index = index @@ -150,12 +152,7 @@ class ScrapeSensor(SensorEntity): self._attr_device_class = device_class self._attr_state_class = state_class - @property - def native_value(self): - """Return the state of the device.""" - return self._state - - def _extract_value(self): + def _extract_value(self) -> Any: """Parse the html extraction in the executor.""" raw_data = BeautifulSoup(self.rest.data, "html.parser") _LOGGER.debug(raw_data) @@ -180,30 +177,26 @@ class ScrapeSensor(SensorEntity): _LOGGER.debug(value) return value - async def async_update(self): + async def async_update(self) -> None: """Get the latest data from the source and updates the state.""" await self.rest.async_update() await self._async_update_from_rest_data() - async def async_added_to_hass(self): + async def async_added_to_hass(self) -> None: """Ensure the data from the initial update is reflected in the state.""" await self._async_update_from_rest_data() - async def _async_update_from_rest_data(self): + async def _async_update_from_rest_data(self) -> None: """Update state from the rest data.""" if self.rest.data is None: _LOGGER.error("Unable to retrieve data for %s", self.name) return - try: - value = await self.hass.async_add_executor_job(self._extract_value) - except IndexError: - _LOGGER.error("Unable to extract data from HTML for %s", self.name) - return + value = await self.hass.async_add_executor_job(self._extract_value) if self._value_template is not None: - self._state = self._value_template.async_render_with_possible_json_value( - value, None + self._attr_native_value = ( + self._value_template.async_render_with_possible_json_value(value, None) ) else: - self._state = value + self._attr_native_value = value diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 325b6f8f554..10bb8f5af4d 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -275,6 +275,9 @@ azure-eventhub==5.7.0 # homeassistant.components.homekit base36==0.1.1 +# homeassistant.components.scrape +beautifulsoup4==4.10.0 + # homeassistant.components.zha bellows==0.29.0 diff --git a/tests/components/scrape/__init__.py b/tests/components/scrape/__init__.py new file mode 100644 index 00000000000..0ba9266a79d --- /dev/null +++ b/tests/components/scrape/__init__.py @@ -0,0 +1,83 @@ +"""Tests for scrape component.""" +from __future__ import annotations + +from typing import Any + + +def return_config( + select, + name, + *, + attribute=None, + index=None, + template=None, + uom=None, + device_class=None, + state_class=None, + authentication=None, + username=None, + password=None, + headers=None, +) -> dict[str, dict[str, Any]]: + """Return config.""" + config = { + "platform": "scrape", + "resource": "https://www.home-assistant.io", + "select": select, + "name": name, + } + if attribute: + config["attribute"] = attribute + if index: + config["index"] = index + if template: + config["value_template"] = template + if uom: + config["unit_of_measurement"] = uom + if device_class: + config["device_class"] = device_class + if state_class: + config["state_class"] = state_class + if authentication: + config["authentication"] = authentication + config["username"] = username + config["password"] = password + if headers: + config["headers"] = headers + return config + + +class MockRestData: + """Mock RestData.""" + + def __init__( + self, + payload, + ): + """Init RestDataMock.""" + self.data: str | None = None + self.payload = payload + self.count = 0 + + async def async_update(self, data: bool | None = True) -> None: + """Update.""" + self.count += 1 + if self.payload == "test_scrape_sensor": + self.data = ( + "
" + "

Current Version: 2021.12.10

Released: January 17, 2022" + "
Release notes
" + "" + ) + if self.payload == "test_scrape_uom_and_classes": + self.data = ( + "
" + "

Current Temperature: 22.1

" + "
Temp check
" + ) + if self.payload == "test_scrape_sensor_authentication": + self.data = "
secret text
" + if self.payload == "test_scrape_sensor_no_data": + self.data = None + if self.count == 3: + self.data = None diff --git a/tests/components/scrape/test_sensor.py b/tests/components/scrape/test_sensor.py new file mode 100644 index 00000000000..aaf156208ef --- /dev/null +++ b/tests/components/scrape/test_sensor.py @@ -0,0 +1,206 @@ +"""The tests for the Scrape sensor platform.""" +from __future__ import annotations + +from unittest.mock import patch + +from homeassistant.components.sensor import SensorDeviceClass, SensorStateClass +from homeassistant.components.sensor.const import CONF_STATE_CLASS +from homeassistant.const import ( + CONF_DEVICE_CLASS, + CONF_UNIT_OF_MEASUREMENT, + STATE_UNKNOWN, + TEMP_CELSIUS, +) +from homeassistant.core import HomeAssistant +from homeassistant.helpers.entity_component import async_update_entity +from homeassistant.setup import async_setup_component + +from . import MockRestData, return_config + +DOMAIN = "scrape" + + +async def test_scrape_sensor(hass: HomeAssistant) -> None: + """Test Scrape sensor minimal.""" + config = {"sensor": return_config(select=".current-version h1", name="HA version")} + + mocker = MockRestData("test_scrape_sensor") + with patch( + "homeassistant.components.scrape.sensor.RestData", + return_value=mocker, + ): + assert await async_setup_component(hass, "sensor", config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.ha_version") + assert state.state == "Current Version: 2021.12.10" + + +async def test_scrape_sensor_value_template(hass: HomeAssistant) -> None: + """Test Scrape sensor with value template.""" + config = { + "sensor": return_config( + select=".current-version h1", + name="HA version", + template="{{ value.split(':')[1] }}", + ) + } + + mocker = MockRestData("test_scrape_sensor") + with patch( + "homeassistant.components.scrape.sensor.RestData", + return_value=mocker, + ): + assert await async_setup_component(hass, "sensor", config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.ha_version") + assert state.state == "2021.12.10" + + +async def test_scrape_uom_and_classes(hass: HomeAssistant) -> None: + """Test Scrape sensor for unit of measurement, device class and state class.""" + config = { + "sensor": return_config( + select=".current-temp h3", + name="Current Temp", + template="{{ value.split(':')[1] }}", + uom="°C", + device_class="temperature", + state_class="measurement", + ) + } + + mocker = MockRestData("test_scrape_uom_and_classes") + with patch( + "homeassistant.components.scrape.sensor.RestData", + return_value=mocker, + ): + assert await async_setup_component(hass, "sensor", config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.current_temp") + assert state.state == "22.1" + assert state.attributes[CONF_UNIT_OF_MEASUREMENT] == TEMP_CELSIUS + assert state.attributes[CONF_DEVICE_CLASS] == SensorDeviceClass.TEMPERATURE + assert state.attributes[CONF_STATE_CLASS] == SensorStateClass.MEASUREMENT + + +async def test_scrape_sensor_authentication(hass: HomeAssistant) -> None: + """Test Scrape sensor with authentication.""" + config = { + "sensor": [ + return_config( + select=".return", + name="Auth page", + username="user@secret.com", + password="12345678", + authentication="digest", + ), + return_config( + select=".return", + name="Auth page2", + username="user@secret.com", + password="12345678", + ), + ] + } + + mocker = MockRestData("test_scrape_sensor_authentication") + with patch( + "homeassistant.components.scrape.sensor.RestData", + return_value=mocker, + ): + assert await async_setup_component(hass, "sensor", config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.auth_page") + assert state.state == "secret text" + state2 = hass.states.get("sensor.auth_page2") + assert state2.state == "secret text" + + +async def test_scrape_sensor_no_data(hass: HomeAssistant) -> None: + """Test Scrape sensor fails on no data.""" + config = {"sensor": return_config(select=".current-version h1", name="HA version")} + + mocker = MockRestData("test_scrape_sensor_no_data") + with patch( + "homeassistant.components.scrape.sensor.RestData", + return_value=mocker, + ): + assert await async_setup_component(hass, "sensor", config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.ha_version") + assert state is None + + +async def test_scrape_sensor_no_data_refresh(hass: HomeAssistant) -> None: + """Test Scrape sensor no data on refresh.""" + config = {"sensor": return_config(select=".current-version h1", name="HA version")} + + mocker = MockRestData("test_scrape_sensor") + with patch( + "homeassistant.components.scrape.sensor.RestData", + return_value=mocker, + ): + assert await async_setup_component(hass, "sensor", config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.ha_version") + assert state + assert state.state == "Current Version: 2021.12.10" + + mocker.data = None + await async_update_entity(hass, "sensor.ha_version") + + assert mocker.data is None + assert state is not None + assert state.state == "Current Version: 2021.12.10" + + +async def test_scrape_sensor_attribute_and_tag(hass: HomeAssistant) -> None: + """Test Scrape sensor with attribute and tag.""" + config = { + "sensor": [ + return_config(select="div", name="HA class", index=1, attribute="class"), + return_config(select="template", name="HA template"), + ] + } + + mocker = MockRestData("test_scrape_sensor") + with patch( + "homeassistant.components.scrape.sensor.RestData", + return_value=mocker, + ): + assert await async_setup_component(hass, "sensor", config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.ha_class") + assert state.state == "['links']" + state2 = hass.states.get("sensor.ha_template") + assert state2.state == "Trying to get" + + +async def test_scrape_sensor_errors(hass: HomeAssistant) -> None: + """Test Scrape sensor handle errors.""" + config = { + "sensor": [ + return_config(select="div", name="HA class", index=5, attribute="class"), + return_config(select="div", name="HA class2", attribute="classes"), + ] + } + + mocker = MockRestData("test_scrape_sensor") + with patch( + "homeassistant.components.scrape.sensor.RestData", + return_value=mocker, + ): + assert await async_setup_component(hass, "sensor", config) + await hass.async_block_till_done() + + state = hass.states.get("sensor.ha_class") + assert state.state == STATE_UNKNOWN + state2 = hass.states.get("sensor.ha_class2") + assert state2.state == STATE_UNKNOWN