evlli/hass-core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Code quality scrape (#65441)

Browse source
This commit is contained in:
G Johansson 2022-02-12 15:28:54 +01:00 • committed by GitHub
parent 62d49dcf98
commit 6d20e68e6d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 334 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.coveragerc
View file

@ -1009,7 +1009,6 @@ omit =
homeassistant/components/samsungtv/diagnostics.py
homeassistant/components/satel_integra/*
homeassistant/components/schluter/*
homeassistant/components/scrape/sensor.py
homeassistant/components/screenlogic/__init__.py
homeassistant/components/screenlogic/binary_sensor.py
homeassistant/components/screenlogic/climate.py

1
CODEOWNERS
View file

@ -808,6 +808,7 @@ homeassistant/components/scene/* @home-assistant/core
tests/components/scene/* @home-assistant/core
homeassistant/components/schluter/* @prairieapps
homeassistant/components/scrape/* @fabaff
tests/components/scrape/* @fabaff
homeassistant/components/screenlogic/* @dieselrabbit @bdraco
tests/components/screenlogic/* @dieselrabbit @bdraco
homeassistant/components/script/* @home-assistant/core

89
homeassistant/components/scrape/sensor.py
View file

@ -2,6 +2,7 @@
from __future__ import annotations
import logging
from typing import Any
from bs4 import BeautifulSoup
import httpx
@ -11,7 +12,7 @@ from homeassistant.components.rest.data import RestData
from homeassistant.components.sensor import (
CONF_STATE_CLASS,
DEVICE_CLASSES_SCHEMA,
PLATFORM_SCHEMA,
PLATFORM_SCHEMA as PARENT_PLATFORM_SCHEMA,
STATE_CLASSES_SCHEMA,
SensorEntity,
)
@ -33,6 +34,7 @@ from homeassistant.core import HomeAssistant
from homeassistant.exceptions import PlatformNotReady
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.template import Template
from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
_LOGGER = logging.getLogger(__name__)
@ -44,7 +46,7 @@ CONF_INDEX = "index"
DEFAULT_NAME = "Web scrape"
DEFAULT_VERIFY_SSL = True
PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
PLATFORM_SCHEMA = PARENT_PLATFORM_SCHEMA.extend(
{
vol.Required(CONF_RESOURCE): cv.string,
vol.Required(CONF_SELECT): cv.string,
@ -73,32 +75,32 @@ async def async_setup_platform(
discovery_info: DiscoveryInfoType | None = None,
) -> None:
"""Set up the Web scrape sensor."""
name = config.get(CONF_NAME)
resource = config.get(CONF_RESOURCE)
method = "GET"
payload = None
headers = config.get(CONF_HEADERS)
verify_ssl = config.get(CONF_VERIFY_SSL)
select = config.get(CONF_SELECT)
attr = config.get(CONF_ATTR)
index = config.get(CONF_INDEX)
unit = config.get(CONF_UNIT_OF_MEASUREMENT)
device_class = config.get(CONF_DEVICE_CLASS)
state_class = config.get(CONF_STATE_CLASS)
username = config.get(CONF_USERNAME)
password = config.get(CONF_PASSWORD)
name: str = config[CONF_NAME]
resource: str = config[CONF_RESOURCE]
method: str = "GET"
payload: str | None = None
headers: str | None = config.get(CONF_HEADERS)
verify_ssl: bool = config[CONF_VERIFY_SSL]
select: str | None = config.get(CONF_SELECT)
attr: str | None = config.get(CONF_ATTR)
index: int = config[CONF_INDEX]
unit: str | None = config.get(CONF_UNIT_OF_MEASUREMENT)
device_class: str | None = config.get(CONF_DEVICE_CLASS)
state_class: str | None = config.get(CONF_STATE_CLASS)
username: str | None = config.get(CONF_USERNAME)
password: str | None = config.get(CONF_PASSWORD)
value_template: Template | None = config.get(CONF_VALUE_TEMPLATE)
if (value_template := config.get(CONF_VALUE_TEMPLATE)) is not None:
if value_template is not None:
value_template.hass = hass
auth: httpx.DigestAuth | tuple[str, str] | None
auth: httpx.DigestAuth | tuple[str, str] | None = None
if username and password:
if config.get(CONF_AUTHENTICATION) == HTTP_DIGEST_AUTHENTICATION:
auth = httpx.DigestAuth(username, password)
else:
auth = (username, password)
else:
auth = None
rest = RestData(hass, method, resource, auth, headers, None, payload, verify_ssl)
await rest.async_update()
@ -128,19 +130,19 @@ class ScrapeSensor(SensorEntity):
def __init__(
self,
rest,
name,
select,
attr,
index,
value_template,
unit,
device_class,
state_class,
):
rest: RestData,
name: str,
select: str | None,
attr: str | None,
index: int,
value_template: Template | None,
unit: str | None,
device_class: str | None,
state_class: str | None,
) -> None:
"""Initialize a web scrape sensor."""
self.rest = rest
self._state = None
self._attr_native_value = None
self._select = select
self._attr = attr
self._index = index
@ -150,12 +152,7 @@ class ScrapeSensor(SensorEntity):
self._attr_device_class = device_class
self._attr_state_class = state_class
@property
def native_value(self):
"""Return the state of the device."""
return self._state
def _extract_value(self):
def _extract_value(self) -> Any:
"""Parse the html extraction in the executor."""
raw_data = BeautifulSoup(self.rest.data, "html.parser")
_LOGGER.debug(raw_data)
@ -180,30 +177,26 @@ class ScrapeSensor(SensorEntity):
_LOGGER.debug(value)
return value
async def async_update(self):
async def async_update(self) -> None:
"""Get the latest data from the source and updates the state."""
await self.rest.async_update()
await self._async_update_from_rest_data()
async def async_added_to_hass(self):
async def async_added_to_hass(self) -> None:
"""Ensure the data from the initial update is reflected in the state."""
await self._async_update_from_rest_data()
async def _async_update_from_rest_data(self):
async def _async_update_from_rest_data(self) -> None:
"""Update state from the rest data."""
if self.rest.data is None:
_LOGGER.error("Unable to retrieve data for %s", self.name)
return
try:
value = await self.hass.async_add_executor_job(self._extract_value)
except IndexError:
_LOGGER.error("Unable to extract data from HTML for %s", self.name)
return
value = await self.hass.async_add_executor_job(self._extract_value)
if self._value_template is not None:
self._state = self._value_template.async_render_with_possible_json_value(
value, None
self._attr_native_value = (
self._value_template.async_render_with_possible_json_value(value, None)
)
else:
self._state = value
self._attr_native_value = value

3
requirements_test_all.txt
View file

@ -275,6 +275,9 @@ azure-eventhub==5.7.0
# homeassistant.components.homekit
base36==0.1.1
# homeassistant.components.scrape
beautifulsoup4==4.10.0
# homeassistant.components.zha
bellows==0.29.0

83
tests/components/scrape/__init__.py Normal file
View file

@ -0,0 +1,83 @@
"""Tests for scrape component."""
from __future__ import annotations
from typing import Any
def return_config(
select,
name,
*,
attribute=None,
index=None,
template=None,
uom=None,
device_class=None,
state_class=None,
authentication=None,
username=None,
password=None,
headers=None,
) -> dict[str, dict[str, Any]]:
"""Return config."""
config = {
"platform": "scrape",
"resource": "https://www.home-assistant.io",
"select": select,
"name": name,
}
if attribute:
config["attribute"] = attribute
if index:
config["index"] = index
if template:
config["value_template"] = template
if uom:
config["unit_of_measurement"] = uom
if device_class:
config["device_class"] = device_class
if state_class:
config["state_class"] = state_class
if authentication:
config["authentication"] = authentication
config["username"] = username
config["password"] = password
if headers:
config["headers"] = headers
return config
class MockRestData:
"""Mock RestData."""
def __init__(
self,
payload,
):
"""Init RestDataMock."""
self.data: str | None = None
self.payload = payload
self.count = 0
async def async_update(self, data: bool | None = True) -> None:
"""Update."""
self.count += 1
if self.payload == "test_scrape_sensor":
self.data = (
"<div class='current-version material-card text'>"
"<h1>Current Version: 2021.12.10</h1>Released: <span class='release-date'>January 17, 2022</span>"
"<div class='links' style='links'><a href='/latest-release-notes/'>Release notes</a></div></div>"
"<template>Trying to get</template>"
)
if self.payload == "test_scrape_uom_and_classes":
self.data = (
"<div class='current-temp temp-card text'>"
"<h3>Current Temperature: 22.1</h3>"
"<div class='links'><a href='/check_temp/'>Temp check</a></div></div>"
)
if self.payload == "test_scrape_sensor_authentication":
self.data = "<div class='return'>secret text</div>"
if self.payload == "test_scrape_sensor_no_data":
self.data = None
if self.count == 3:
self.data = None

206
tests/components/scrape/test_sensor.py Normal file
View file

@ -0,0 +1,206 @@
"""The tests for the Scrape sensor platform."""
from __future__ import annotations
from unittest.mock import patch
from homeassistant.components.sensor import SensorDeviceClass, SensorStateClass
from homeassistant.components.sensor.const import CONF_STATE_CLASS
from homeassistant.const import (
CONF_DEVICE_CLASS,
CONF_UNIT_OF_MEASUREMENT,
STATE_UNKNOWN,
TEMP_CELSIUS,
)
from homeassistant.core import HomeAssistant
from homeassistant.helpers.entity_component import async_update_entity
from homeassistant.setup import async_setup_component
from . import MockRestData, return_config
DOMAIN = "scrape"
async def test_scrape_sensor(hass: HomeAssistant) -> None:
"""Test Scrape sensor minimal."""
config = {"sensor": return_config(select=".current-version h1", name="HA version")}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.scrape.sensor.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state.state == "Current Version: 2021.12.10"
async def test_scrape_sensor_value_template(hass: HomeAssistant) -> None:
"""Test Scrape sensor with value template."""
config = {
"sensor": return_config(
select=".current-version h1",
name="HA version",
template="{{ value.split(':')[1] }}",
)
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.scrape.sensor.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state.state == "2021.12.10"
async def test_scrape_uom_and_classes(hass: HomeAssistant) -> None:
"""Test Scrape sensor for unit of measurement, device class and state class."""
config = {
"sensor": return_config(
select=".current-temp h3",
name="Current Temp",
template="{{ value.split(':')[1] }}",
uom="°C",
device_class="temperature",
state_class="measurement",
)
}
mocker = MockRestData("test_scrape_uom_and_classes")
with patch(
"homeassistant.components.scrape.sensor.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.current_temp")
assert state.state == "22.1"
assert state.attributes[CONF_UNIT_OF_MEASUREMENT] == TEMP_CELSIUS
assert state.attributes[CONF_DEVICE_CLASS] == SensorDeviceClass.TEMPERATURE
assert state.attributes[CONF_STATE_CLASS] == SensorStateClass.MEASUREMENT
async def test_scrape_sensor_authentication(hass: HomeAssistant) -> None:
"""Test Scrape sensor with authentication."""
config = {
"sensor": [
return_config(
select=".return",
name="Auth page",
username="user@secret.com",
password="12345678",
authentication="digest",
),
return_config(
select=".return",
name="Auth page2",
username="user@secret.com",
password="12345678",
),
]
}
mocker = MockRestData("test_scrape_sensor_authentication")
with patch(
"homeassistant.components.scrape.sensor.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.auth_page")
assert state.state == "secret text"
state2 = hass.states.get("sensor.auth_page2")
assert state2.state == "secret text"
async def test_scrape_sensor_no_data(hass: HomeAssistant) -> None:
"""Test Scrape sensor fails on no data."""
config = {"sensor": return_config(select=".current-version h1", name="HA version")}
mocker = MockRestData("test_scrape_sensor_no_data")
with patch(
"homeassistant.components.scrape.sensor.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state is None
async def test_scrape_sensor_no_data_refresh(hass: HomeAssistant) -> None:
"""Test Scrape sensor no data on refresh."""
config = {"sensor": return_config(select=".current-version h1", name="HA version")}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.scrape.sensor.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_version")
assert state
assert state.state == "Current Version: 2021.12.10"
mocker.data = None
await async_update_entity(hass, "sensor.ha_version")
assert mocker.data is None
assert state is not None
assert state.state == "Current Version: 2021.12.10"
async def test_scrape_sensor_attribute_and_tag(hass: HomeAssistant) -> None:
"""Test Scrape sensor with attribute and tag."""
config = {
"sensor": [
return_config(select="div", name="HA class", index=1, attribute="class"),
return_config(select="template", name="HA template"),
]
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.scrape.sensor.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_class")
assert state.state == "['links']"
state2 = hass.states.get("sensor.ha_template")
assert state2.state == "Trying to get"
async def test_scrape_sensor_errors(hass: HomeAssistant) -> None:
"""Test Scrape sensor handle errors."""
config = {
"sensor": [
return_config(select="div", name="HA class", index=5, attribute="class"),
return_config(select="div", name="HA class2", attribute="classes"),
]
}
mocker = MockRestData("test_scrape_sensor")
with patch(
"homeassistant.components.scrape.sensor.RestData",
return_value=mocker,
):
assert await async_setup_component(hass, "sensor", config)
await hass.async_block_till_done()
state = hass.states.get("sensor.ha_class")
assert state.state == STATE_UNKNOWN
state2 = hass.states.get("sensor.ha_class2")
assert state2.state == STATE_UNKNOWN