Code quality scrape (#65441)

.coveragerc

@@ -1009,7 +1009,6 @@ omit =
     homeassistant/components/samsungtv/diagnostics.py
     homeassistant/components/satel_integra/*
     homeassistant/components/schluter/*
-    homeassistant/components/scrape/sensor.py
     homeassistant/components/screenlogic/__init__.py
     homeassistant/components/screenlogic/binary_sensor.py
     homeassistant/components/screenlogic/climate.py

CODEOWNERS

@@ -808,6 +808,7 @@ homeassistant/components/scene/* @home-assistant/core
 tests/components/scene/* @home-assistant/core
 homeassistant/components/schluter/* @prairieapps
 homeassistant/components/scrape/* @fabaff
+tests/components/scrape/* @fabaff
 homeassistant/components/screenlogic/* @dieselrabbit @bdraco
 tests/components/screenlogic/* @dieselrabbit @bdraco
 homeassistant/components/script/* @home-assistant/core

homeassistant/components/scrape/sensor.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import logging
+from typing import Any
 
 from bs4 import BeautifulSoup
 import httpx
@@ -11,7 +12,7 @@ from homeassistant.components.rest.data import RestData
 from homeassistant.components.sensor import (
     CONF_STATE_CLASS,
     DEVICE_CLASSES_SCHEMA,
-    PLATFORM_SCHEMA,
+    PLATFORM_SCHEMA as PARENT_PLATFORM_SCHEMA,
     STATE_CLASSES_SCHEMA,
     SensorEntity,
 )
@@ -33,6 +34,7 @@ from homeassistant.core import HomeAssistant
 from homeassistant.exceptions import PlatformNotReady
 import homeassistant.helpers.config_validation as cv
 from homeassistant.helpers.entity_platform import AddEntitiesCallback
+from homeassistant.helpers.template import Template
 from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
 
 _LOGGER = logging.getLogger(__name__)
@@ -44,7 +46,7 @@ CONF_INDEX = "index"
 DEFAULT_NAME = "Web scrape"
 DEFAULT_VERIFY_SSL = True
 
-PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
+PLATFORM_SCHEMA = PARENT_PLATFORM_SCHEMA.extend(
     {
         vol.Required(CONF_RESOURCE): cv.string,
         vol.Required(CONF_SELECT): cv.string,
@@ -73,32 +75,32 @@ async def async_setup_platform(
     discovery_info: DiscoveryInfoType | None = None,
 ) -> None:
     """Set up the Web scrape sensor."""
-    name = config.get(CONF_NAME)
+    name: str = config[CONF_NAME]
-    resource = config.get(CONF_RESOURCE)
+    resource: str = config[CONF_RESOURCE]
-    method = "GET"
+    method: str = "GET"
-    payload = None
+    payload: str | None = None
-    headers = config.get(CONF_HEADERS)
+    headers: str | None = config.get(CONF_HEADERS)
-    verify_ssl = config.get(CONF_VERIFY_SSL)
+    verify_ssl: bool = config[CONF_VERIFY_SSL]
-    select = config.get(CONF_SELECT)
+    select: str | None = config.get(CONF_SELECT)
-    attr = config.get(CONF_ATTR)
+    attr: str | None = config.get(CONF_ATTR)
-    index = config.get(CONF_INDEX)
+    index: int = config[CONF_INDEX]
-    unit = config.get(CONF_UNIT_OF_MEASUREMENT)
+    unit: str | None = config.get(CONF_UNIT_OF_MEASUREMENT)
-    device_class = config.get(CONF_DEVICE_CLASS)
+    device_class: str | None = config.get(CONF_DEVICE_CLASS)
-    state_class = config.get(CONF_STATE_CLASS)
+    state_class: str | None = config.get(CONF_STATE_CLASS)
-    username = config.get(CONF_USERNAME)
+    username: str | None = config.get(CONF_USERNAME)
-    password = config.get(CONF_PASSWORD)
+    password: str | None = config.get(CONF_PASSWORD)
+    value_template: Template | None = config.get(CONF_VALUE_TEMPLATE)
 
-    if (value_template := config.get(CONF_VALUE_TEMPLATE)) is not None:
+    if value_template is not None:
         value_template.hass = hass
 
-    auth: httpx.DigestAuth | tuple[str, str] | None
+    auth: httpx.DigestAuth | tuple[str, str] | None = None
     if username and password:
         if config.get(CONF_AUTHENTICATION) == HTTP_DIGEST_AUTHENTICATION:
             auth = httpx.DigestAuth(username, password)
         else:
             auth = (username, password)
-    else:
+
-        auth = None
     rest = RestData(hass, method, resource, auth, headers, None, payload, verify_ssl)
     await rest.async_update()
 
@@ -128,19 +130,19 @@ class ScrapeSensor(SensorEntity):
 
     def __init__(
         self,
-        rest,
+        rest: RestData,
-        name,
+        name: str,
-        select,
+        select: str | None,
-        attr,
+        attr: str | None,
-        index,
+        index: int,
-        value_template,
+        value_template: Template | None,
-        unit,
+        unit: str | None,
-        device_class,
+        device_class: str | None,
-        state_class,
+        state_class: str | None,
-    ):
+    ) -> None:
         """Initialize a web scrape sensor."""
         self.rest = rest
-        self._state = None
+        self._attr_native_value = None
         self._select = select
         self._attr = attr
         self._index = index
@@ -150,12 +152,7 @@ class ScrapeSensor(SensorEntity):
         self._attr_device_class = device_class
         self._attr_state_class = state_class
 
-    @property
+    def _extract_value(self) -> Any:
-    def native_value(self):
-        """Return the state of the device."""
-        return self._state
-
-    def _extract_value(self):
         """Parse the html extraction in the executor."""
         raw_data = BeautifulSoup(self.rest.data, "html.parser")
         _LOGGER.debug(raw_data)
@@ -180,30 +177,26 @@ class ScrapeSensor(SensorEntity):
         _LOGGER.debug(value)
         return value
 
-    async def async_update(self):
+    async def async_update(self) -> None:
         """Get the latest data from the source and updates the state."""
         await self.rest.async_update()
         await self._async_update_from_rest_data()
 
-    async def async_added_to_hass(self):
+    async def async_added_to_hass(self) -> None:
         """Ensure the data from the initial update is reflected in the state."""
         await self._async_update_from_rest_data()
 
-    async def _async_update_from_rest_data(self):
+    async def _async_update_from_rest_data(self) -> None:
         """Update state from the rest data."""
         if self.rest.data is None:
             _LOGGER.error("Unable to retrieve data for %s", self.name)
             return
 
-        try:
+        value = await self.hass.async_add_executor_job(self._extract_value)
-            value = await self.hass.async_add_executor_job(self._extract_value)
-        except IndexError:
-            _LOGGER.error("Unable to extract data from HTML for %s", self.name)
-            return
 
         if self._value_template is not None:
-            self._state = self._value_template.async_render_with_possible_json_value(
+            self._attr_native_value = (
-                value, None
+                self._value_template.async_render_with_possible_json_value(value, None)
             )
         else:
-            self._state = value
+            self._attr_native_value = value

requirements_test_all.txt

@@ -275,6 +275,9 @@ azure-eventhub==5.7.0
 # homeassistant.components.homekit
 base36==0.1.1
 
+# homeassistant.components.scrape
+beautifulsoup4==4.10.0
+
 # homeassistant.components.zha
 bellows==0.29.0
 

tests/components/scrape/__init__.py

@@ -0,0 +1,83 @@
+"""Tests for scrape component."""
+from __future__ import annotations
+
+from typing import Any
+
+
+def return_config(
+    select,
+    name,
+    *,
+    attribute=None,
+    index=None,
+    template=None,
+    uom=None,
+    device_class=None,
+    state_class=None,
+    authentication=None,
+    username=None,
+    password=None,
+    headers=None,
+) -> dict[str, dict[str, Any]]:
+    """Return config."""
+    config = {
+        "platform": "scrape",
+        "resource": "https://www.home-assistant.io",
+        "select": select,
+        "name": name,
+    }
+    if attribute:
+        config["attribute"] = attribute
+    if index:
+        config["index"] = index
+    if template:
+        config["value_template"] = template
+    if uom:
+        config["unit_of_measurement"] = uom
+    if device_class:
+        config["device_class"] = device_class
+    if state_class:
+        config["state_class"] = state_class
+    if authentication:
+        config["authentication"] = authentication
+        config["username"] = username
+        config["password"] = password
+    if headers:
+        config["headers"] = headers
+    return config
+
+
+class MockRestData:
+    """Mock RestData."""
+
+    def __init__(
+        self,
+        payload,
+    ):
+        """Init RestDataMock."""
+        self.data: str | None = None
+        self.payload = payload
+        self.count = 0
+
+    async def async_update(self, data: bool | None = True) -> None:
+        """Update."""
+        self.count += 1
+        if self.payload == "test_scrape_sensor":
+            self.data = (
+                "<div class='current-version material-card text'>"
+                "<h1>Current Version: 2021.12.10</h1>Released: <span class='release-date'>January 17, 2022</span>"
+                "<div class='links' style='links'><a href='/latest-release-notes/'>Release notes</a></div></div>"
+                "<template>Trying to get</template>"
+            )
+        if self.payload == "test_scrape_uom_and_classes":
+            self.data = (
+                "<div class='current-temp temp-card text'>"
+                "<h3>Current Temperature: 22.1</h3>"
+                "<div class='links'><a href='/check_temp/'>Temp check</a></div></div>"
+            )
+        if self.payload == "test_scrape_sensor_authentication":
+            self.data = "<div class='return'>secret text</div>"
+        if self.payload == "test_scrape_sensor_no_data":
+            self.data = None
+        if self.count == 3:
+            self.data = None

tests/components/scrape/test_sensor.py

@@ -0,0 +1,206 @@
+"""The tests for the Scrape sensor platform."""
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from homeassistant.components.sensor import SensorDeviceClass, SensorStateClass
+from homeassistant.components.sensor.const import CONF_STATE_CLASS
+from homeassistant.const import (
+    CONF_DEVICE_CLASS,
+    CONF_UNIT_OF_MEASUREMENT,
+    STATE_UNKNOWN,
+    TEMP_CELSIUS,
+)
+from homeassistant.core import HomeAssistant
+from homeassistant.helpers.entity_component import async_update_entity
+from homeassistant.setup import async_setup_component
+
+from . import MockRestData, return_config
+
+DOMAIN = "scrape"
+
+
+async def test_scrape_sensor(hass: HomeAssistant) -> None:
+    """Test Scrape sensor minimal."""
+    config = {"sensor": return_config(select=".current-version h1", name="HA version")}
+
+    mocker = MockRestData("test_scrape_sensor")
+    with patch(
+        "homeassistant.components.scrape.sensor.RestData",
+        return_value=mocker,
+    ):
+        assert await async_setup_component(hass, "sensor", config)
+        await hass.async_block_till_done()
+
+    state = hass.states.get("sensor.ha_version")
+    assert state.state == "Current Version: 2021.12.10"
+
+
+async def test_scrape_sensor_value_template(hass: HomeAssistant) -> None:
+    """Test Scrape sensor with value template."""
+    config = {
+        "sensor": return_config(
+            select=".current-version h1",
+            name="HA version",
+            template="{{ value.split(':')[1] }}",
+        )
+    }
+
+    mocker = MockRestData("test_scrape_sensor")
+    with patch(
+        "homeassistant.components.scrape.sensor.RestData",
+        return_value=mocker,
+    ):
+        assert await async_setup_component(hass, "sensor", config)
+        await hass.async_block_till_done()
+
+    state = hass.states.get("sensor.ha_version")
+    assert state.state == "2021.12.10"
+
+
+async def test_scrape_uom_and_classes(hass: HomeAssistant) -> None:
+    """Test Scrape sensor for unit of measurement, device class and state class."""
+    config = {
+        "sensor": return_config(
+            select=".current-temp h3",
+            name="Current Temp",
+            template="{{ value.split(':')[1] }}",
+            uom="°C",
+            device_class="temperature",
+            state_class="measurement",
+        )
+    }
+
+    mocker = MockRestData("test_scrape_uom_and_classes")
+    with patch(
+        "homeassistant.components.scrape.sensor.RestData",
+        return_value=mocker,
+    ):
+        assert await async_setup_component(hass, "sensor", config)
+        await hass.async_block_till_done()
+
+    state = hass.states.get("sensor.current_temp")
+    assert state.state == "22.1"
+    assert state.attributes[CONF_UNIT_OF_MEASUREMENT] == TEMP_CELSIUS
+    assert state.attributes[CONF_DEVICE_CLASS] == SensorDeviceClass.TEMPERATURE
+    assert state.attributes[CONF_STATE_CLASS] == SensorStateClass.MEASUREMENT
+
+
+async def test_scrape_sensor_authentication(hass: HomeAssistant) -> None:
+    """Test Scrape sensor with authentication."""
+    config = {
+        "sensor": [
+            return_config(
+                select=".return",
+                name="Auth page",
+                username="user@secret.com",
+                password="12345678",
+                authentication="digest",
+            ),
+            return_config(
+                select=".return",
+                name="Auth page2",
+                username="user@secret.com",
+                password="12345678",
+            ),
+        ]
+    }
+
+    mocker = MockRestData("test_scrape_sensor_authentication")
+    with patch(
+        "homeassistant.components.scrape.sensor.RestData",
+        return_value=mocker,
+    ):
+        assert await async_setup_component(hass, "sensor", config)
+        await hass.async_block_till_done()
+
+    state = hass.states.get("sensor.auth_page")
+    assert state.state == "secret text"
+    state2 = hass.states.get("sensor.auth_page2")
+    assert state2.state == "secret text"
+
+
+async def test_scrape_sensor_no_data(hass: HomeAssistant) -> None:
+    """Test Scrape sensor fails on no data."""
+    config = {"sensor": return_config(select=".current-version h1", name="HA version")}
+
+    mocker = MockRestData("test_scrape_sensor_no_data")
+    with patch(
+        "homeassistant.components.scrape.sensor.RestData",
+        return_value=mocker,
+    ):
+        assert await async_setup_component(hass, "sensor", config)
+        await hass.async_block_till_done()
+
+    state = hass.states.get("sensor.ha_version")
+    assert state is None
+
+
+async def test_scrape_sensor_no_data_refresh(hass: HomeAssistant) -> None:
+    """Test Scrape sensor no data on refresh."""
+    config = {"sensor": return_config(select=".current-version h1", name="HA version")}
+
+    mocker = MockRestData("test_scrape_sensor")
+    with patch(
+        "homeassistant.components.scrape.sensor.RestData",
+        return_value=mocker,
+    ):
+        assert await async_setup_component(hass, "sensor", config)
+        await hass.async_block_till_done()
+
+    state = hass.states.get("sensor.ha_version")
+    assert state
+    assert state.state == "Current Version: 2021.12.10"
+
+    mocker.data = None
+    await async_update_entity(hass, "sensor.ha_version")
+
+    assert mocker.data is None
+    assert state is not None
+    assert state.state == "Current Version: 2021.12.10"
+
+
+async def test_scrape_sensor_attribute_and_tag(hass: HomeAssistant) -> None:
+    """Test Scrape sensor with attribute and tag."""
+    config = {
+        "sensor": [
+            return_config(select="div", name="HA class", index=1, attribute="class"),
+            return_config(select="template", name="HA template"),
+        ]
+    }
+
+    mocker = MockRestData("test_scrape_sensor")
+    with patch(
+        "homeassistant.components.scrape.sensor.RestData",
+        return_value=mocker,
+    ):
+        assert await async_setup_component(hass, "sensor", config)
+        await hass.async_block_till_done()
+
+    state = hass.states.get("sensor.ha_class")
+    assert state.state == "['links']"
+    state2 = hass.states.get("sensor.ha_template")
+    assert state2.state == "Trying to get"
+
+
+async def test_scrape_sensor_errors(hass: HomeAssistant) -> None:
+    """Test Scrape sensor handle errors."""
+    config = {
+        "sensor": [
+            return_config(select="div", name="HA class", index=5, attribute="class"),
+            return_config(select="div", name="HA class2", attribute="classes"),
+        ]
+    }
+
+    mocker = MockRestData("test_scrape_sensor")
+    with patch(
+        "homeassistant.components.scrape.sensor.RestData",
+        return_value=mocker,
+    ):
+        assert await async_setup_component(hass, "sensor", config)
+        await hass.async_block_till_done()
+
+    state = hass.states.get("sensor.ha_class")
+    assert state.state == STATE_UNKNOWN
+    state2 = hass.states.get("sensor.ha_class2")
+    assert state2.state == STATE_UNKNOWN