diff --git a/homeassistant/components/rest/manifest.json b/homeassistant/components/rest/manifest.json index 8c8b7f39609..fd7eea12f7e 100644 --- a/homeassistant/components/rest/manifest.json +++ b/homeassistant/components/rest/manifest.json @@ -2,7 +2,7 @@ "domain": "rest", "name": "RESTful", "documentation": "https://www.home-assistant.io/integrations/rest", - "requirements": [], + "requirements": ["jsonpath==0.82", "xmltodict==0.12.0"], "dependencies": [], "codeowners": [] } diff --git a/homeassistant/components/rest/sensor.py b/homeassistant/components/rest/sensor.py index 51120cb350c..36fd27c29a5 100644 --- a/homeassistant/components/rest/sensor.py +++ b/homeassistant/components/rest/sensor.py @@ -1,10 +1,13 @@ """Support for RESTful API sensors.""" import json import logging +from xml.parsers.expat import ExpatError +from jsonpath import jsonpath import requests from requests.auth import HTTPBasicAuth, HTTPDigestAuth import voluptuous as vol +import xmltodict from homeassistant.components.sensor import DEVICE_CLASSES_SCHEMA, PLATFORM_SCHEMA from homeassistant.const import ( @@ -38,7 +41,9 @@ DEFAULT_VERIFY_SSL = True DEFAULT_FORCE_UPDATE = False DEFAULT_TIMEOUT = 10 + CONF_JSON_ATTRS = "json_attributes" +CONF_JSON_ATTRS_PATH = "json_attributes_path" METHODS = ["POST", "GET"] PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( @@ -57,6 +62,7 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend( vol.Optional(CONF_UNIT_OF_MEASUREMENT): cv.string, vol.Optional(CONF_DEVICE_CLASS): DEVICE_CLASSES_SCHEMA, vol.Optional(CONF_USERNAME): cv.string, + vol.Optional(CONF_JSON_ATTRS_PATH): cv.string, vol.Optional(CONF_VALUE_TEMPLATE): cv.template, vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean, vol.Optional(CONF_FORCE_UPDATE, default=DEFAULT_FORCE_UPDATE): cv.boolean, @@ -84,6 +90,7 @@ def setup_platform(hass, config, add_entities, discovery_info=None): device_class = config.get(CONF_DEVICE_CLASS) value_template = config.get(CONF_VALUE_TEMPLATE) json_attrs = config.get(CONF_JSON_ATTRS) + json_attrs_path = config.get(CONF_JSON_ATTRS_PATH) force_update = config.get(CONF_FORCE_UPDATE) timeout = config.get(CONF_TIMEOUT) @@ -120,6 +127,7 @@ def setup_platform(hass, config, add_entities, discovery_info=None): json_attrs, force_update, resource_template, + json_attrs_path, ) ], True, @@ -140,6 +148,7 @@ class RestSensor(Entity): json_attrs, force_update, resource_template, + json_attrs_path, ): """Initialize the REST sensor.""" self._hass = hass @@ -153,6 +162,7 @@ class RestSensor(Entity): self._attributes = None self._force_update = force_update self._resource_template = resource_template + self._json_attrs_path = json_attrs_path @property def name(self): @@ -191,12 +201,27 @@ class RestSensor(Entity): self.rest.update() value = self.rest.data + content_type = self.rest.headers.get("content-type") + + if content_type and content_type.startswith("text/xml"): + try: + value = json.dumps(xmltodict.parse(value)) + except ExpatError: + _LOGGER.warning( + "REST xml result could not be parsed and converted to JSON." + ) + _LOGGER.debug("Erroneous XML: %s", value) if self._json_attrs: self._attributes = {} if value: try: json_dict = json.loads(value) + if self._json_attrs_path is not None: + json_dict = jsonpath(json_dict, self._json_attrs_path) + # jsonpath will always store the result in json_dict[0] + # so the next line happens to work exactly as needed to + # find the result if isinstance(json_dict, list): json_dict = json_dict[0] if isinstance(json_dict, dict): @@ -240,6 +265,7 @@ class RestData: self._verify_ssl = verify_ssl self._timeout = timeout self.data = None + self.headers = None def set_url(self, url): """Set url.""" @@ -259,6 +285,8 @@ class RestData: verify=self._verify_ssl, ) self.data = response.text + self.headers = response.headers except requests.exceptions.RequestException as ex: _LOGGER.error("Error fetching data: %s failed with %s", self._resource, ex) self.data = None + self.headers = None diff --git a/requirements_all.txt b/requirements_all.txt index e8f6342d301..3a39bca9e8e 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -742,6 +742,7 @@ iperf3==0.1.11 # homeassistant.components.route53 ipify==1.0.0 +# homeassistant.components.rest # homeassistant.components.verisure jsonpath==0.82 @@ -2097,6 +2098,7 @@ xfinity-gateway==0.0.4 xknx==0.11.2 # homeassistant.components.bluesound +# homeassistant.components.rest # homeassistant.components.startca # homeassistant.components.ted5000 # homeassistant.components.yr diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 3e8eca8d3ac..c502887f2c7 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -278,6 +278,7 @@ iaqualink==0.3.1 # homeassistant.components.influxdb influxdb==5.2.3 +# homeassistant.components.rest # homeassistant.components.verisure jsonpath==0.82 @@ -711,6 +712,7 @@ withings-api==2.1.3 wled==0.2.1 # homeassistant.components.bluesound +# homeassistant.components.rest # homeassistant.components.startca # homeassistant.components.ted5000 # homeassistant.components.yr diff --git a/tests/components/rest/test_sensor.py b/tests/components/rest/test_sensor.py index 7e03eb0fd41..30eeae9a8e3 100644 --- a/tests/components/rest/test_sensor.py +++ b/tests/components/rest/test_sensor.py @@ -6,6 +6,7 @@ import pytest from pytest import raises import requests from requests.exceptions import RequestException, Timeout +from requests.structures import CaseInsensitiveDict import requests_mock import homeassistant.components.rest.sensor as rest @@ -166,6 +167,33 @@ class TestRestSensorSetup(unittest.TestCase): ) assert 2 == mock_req.call_count + @requests_mock.Mocker() + def test_setup_get_xml(self, mock_req): + """Test setup with valid configuration.""" + mock_req.get("http://localhost", status_code=200) + with assert_setup_component(1, "sensor"): + assert setup_component( + self.hass, + "sensor", + { + "sensor": { + "platform": "rest", + "resource": "http://localhost", + "method": "GET", + "value_template": "{{ value_json.key }}", + "name": "foo", + "unit_of_measurement": DATA_MEGABYTES, + "verify_ssl": "true", + "timeout": 30, + "authentication": "basic", + "username": "my username", + "password": "my password", + "headers": {"Accept": "text/xml"}, + } + }, + ) + assert 2 == mock_req.call_count + class TestRestSensor(unittest.TestCase): """Tests for REST sensor platform.""" @@ -178,13 +206,15 @@ class TestRestSensor(unittest.TestCase): self.rest.update = Mock( "rest.RestData.update", side_effect=self.update_side_effect( - '{ "key": "' + self.initial_state + '" }' + '{ "key": "' + self.initial_state + '" }', + CaseInsensitiveDict({"Content-Type": "application/json"}), ), ) self.name = "foo" self.unit_of_measurement = DATA_MEGABYTES self.device_class = None self.value_template = template("{{ value_json.key }}") + self.json_attrs_path = None self.value_template.hass = self.hass self.force_update = False self.resource_template = None @@ -199,15 +229,17 @@ class TestRestSensor(unittest.TestCase): [], self.force_update, self.resource_template, + self.json_attrs_path, ) def tearDown(self): """Stop everything that was started.""" self.hass.stop() - def update_side_effect(self, data): + def update_side_effect(self, data, headers): """Side effect function for mocking RestData.update().""" self.rest.data = data + self.rest.headers = headers def test_name(self): """Test the name.""" @@ -229,7 +261,8 @@ class TestRestSensor(unittest.TestCase): def test_update_when_value_is_none(self): """Test state gets updated to unknown when sensor returns no data.""" self.rest.update = Mock( - "rest.RestData.update", side_effect=self.update_side_effect(None) + "rest.RestData.update", + side_effect=self.update_side_effect(None, CaseInsensitiveDict()), ) self.sensor.update() assert self.sensor.state is None @@ -239,7 +272,10 @@ class TestRestSensor(unittest.TestCase): """Test state gets updated when sensor returns a new status.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect('{ "key": "updated_state" }'), + side_effect=self.update_side_effect( + '{ "key": "updated_state" }', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), ) self.sensor.update() assert "updated_state" == self.sensor.state @@ -248,7 +284,10 @@ class TestRestSensor(unittest.TestCase): def test_update_with_no_template(self): """Test update when there is no value template.""" self.rest.update = Mock( - "rest.RestData.update", side_effect=self.update_side_effect("plain_state") + "rest.RestData.update", + side_effect=self.update_side_effect( + "plain_state", CaseInsensitiveDict({"Content-Type": "application/json"}) + ), ) self.sensor = rest.RestSensor( self.hass, @@ -260,6 +299,7 @@ class TestRestSensor(unittest.TestCase): [], self.force_update, self.resource_template, + self.json_attrs_path, ) self.sensor.update() assert "plain_state" == self.sensor.state @@ -269,7 +309,10 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a JSON result.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect('{ "key": "some_json_value" }'), + side_effect=self.update_side_effect( + '{ "key": "some_json_value" }', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), ) self.sensor = rest.RestSensor( self.hass, @@ -281,6 +324,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.json_attrs_path, ) self.sensor.update() assert "some_json_value" == self.sensor.device_state_attributes["key"] @@ -289,7 +333,10 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a JSON list[0] result.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect('[{ "key": "another_value" }]'), + side_effect=self.update_side_effect( + '[{ "key": "another_value" }]', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), ) self.sensor = rest.RestSensor( self.hass, @@ -301,6 +348,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.json_attrs_path, ) self.sensor.update() assert "another_value" == self.sensor.device_state_attributes["key"] @@ -309,7 +357,10 @@ class TestRestSensor(unittest.TestCase): def test_update_with_json_attrs_no_data(self, mock_logger): """Test attributes when no JSON result fetched.""" self.rest.update = Mock( - "rest.RestData.update", side_effect=self.update_side_effect(None) + "rest.RestData.update", + side_effect=self.update_side_effect( + None, CaseInsensitiveDict({"Content-Type": "application/json"}) + ), ) self.sensor = rest.RestSensor( self.hass, @@ -321,6 +372,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.json_attrs_path, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -331,7 +383,10 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a JSON result.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect('["list", "of", "things"]'), + side_effect=self.update_side_effect( + '["list", "of", "things"]', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), ) self.sensor = rest.RestSensor( self.hass, @@ -343,6 +398,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.json_attrs_path, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -353,7 +409,10 @@ class TestRestSensor(unittest.TestCase): """Test attributes get extracted from a JSON result.""" self.rest.update = Mock( "rest.RestData.update", - side_effect=self.update_side_effect("This is text rather than JSON data."), + side_effect=self.update_side_effect( + "This is text rather than JSON data.", + CaseInsensitiveDict({"Content-Type": "text/plain"}), + ), ) self.sensor = rest.RestSensor( self.hass, @@ -365,6 +424,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.json_attrs_path, ) self.sensor.update() assert {} == self.sensor.device_state_attributes @@ -376,7 +436,8 @@ class TestRestSensor(unittest.TestCase): self.rest.update = Mock( "rest.RestData.update", side_effect=self.update_side_effect( - '{ "key": "json_state_updated_value" }' + '{ "key": "json_state_updated_value" }', + CaseInsensitiveDict({"Content-Type": "application/json"}), ), ) self.sensor = rest.RestSensor( @@ -389,6 +450,7 @@ class TestRestSensor(unittest.TestCase): ["key"], self.force_update, self.resource_template, + self.json_attrs_path, ) self.sensor.update() @@ -397,6 +459,136 @@ class TestRestSensor(unittest.TestCase): "json_state_updated_value" == self.sensor.device_state_attributes["key"] ), self.force_update + def test_update_with_json_attrs_with_json_attrs_path(self): + """Test attributes get extracted from a JSON result with a template for the attributes.""" + json_attrs_path = "$.toplevel.second_level" + value_template = template("{{ value_json.toplevel.master_value }}") + value_template.hass = self.hass + + self.rest.update = Mock( + "rest.RestData.update", + side_effect=self.update_side_effect( + '{ "toplevel": {"master_value": "master", "second_level": {"some_json_key": "some_json_value", "some_json_key2": "some_json_value2" } } }', + CaseInsensitiveDict({"Content-Type": "application/json"}), + ), + ) + self.sensor = rest.RestSensor( + self.hass, + self.rest, + self.name, + self.unit_of_measurement, + self.device_class, + value_template, + ["some_json_key", "some_json_key2"], + self.force_update, + self.resource_template, + json_attrs_path, + ) + + self.sensor.update() + assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"] + assert ( + "some_json_value2" == self.sensor.device_state_attributes["some_json_key2"] + ) + assert "master" == self.sensor.state + + def test_update_with_xml_convert_json_attrs_with_json_attrs_path(self): + """Test attributes get extracted from a JSON result that was converted from XML with a template for the attributes.""" + json_attrs_path = "$.toplevel.second_level" + value_template = template("{{ value_json.toplevel.master_value }}") + value_template.hass = self.hass + + self.rest.update = Mock( + "rest.RestData.update", + side_effect=self.update_side_effect( + "mastersome_json_valuesome_json_value2", + CaseInsensitiveDict({"Content-Type": "text/xml+svg"}), + ), + ) + self.sensor = rest.RestSensor( + self.hass, + self.rest, + self.name, + self.unit_of_measurement, + self.device_class, + value_template, + ["some_json_key", "some_json_key2"], + self.force_update, + self.resource_template, + json_attrs_path, + ) + + self.sensor.update() + assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"] + assert ( + "some_json_value2" == self.sensor.device_state_attributes["some_json_key2"] + ) + assert "master" == self.sensor.state + + def test_update_with_xml_convert_json_attrs_with_jsonattr_template(self): + """Test attributes get extracted from a JSON result that was converted from XML.""" + json_attrs_path = "$.response" + value_template = template("{{ value_json.response.bss.wlan }}") + value_template.hass = self.hass + + self.rest.update = Mock( + "rest.RestData.update", + side_effect=self.update_side_effect( + '01255648alexander000bogus000000000upupupup000x0XF0x0XF 0', + CaseInsensitiveDict({"Content-Type": "text/xml"}), + ), + ) + self.sensor = rest.RestSensor( + self.hass, + self.rest, + self.name, + self.unit_of_measurement, + self.device_class, + value_template, + ["led0", "led1", "temp0", "time0", "ver"], + self.force_update, + self.resource_template, + json_attrs_path, + ) + + self.sensor.update() + assert "0" == self.sensor.device_state_attributes["led0"] + assert "0" == self.sensor.device_state_attributes["led1"] + assert "0x0XF0x0XF" == self.sensor.device_state_attributes["temp0"] + assert "0" == self.sensor.device_state_attributes["time0"] + assert "12556" == self.sensor.device_state_attributes["ver"] + assert "bogus" == self.sensor.state + + @patch("homeassistant.components.rest.sensor._LOGGER") + def test_update_with_xml_convert_bad_xml(self, mock_logger): + """Test attributes get extracted from a XML result with bad xml.""" + value_template = template("{{ value_json.toplevel.master_value }}") + value_template.hass = self.hass + + self.rest.update = Mock( + "rest.RestData.update", + side_effect=self.update_side_effect( + "this is not xml", CaseInsensitiveDict({"Content-Type": "text/xml"}) + ), + ) + self.sensor = rest.RestSensor( + self.hass, + self.rest, + self.name, + self.unit_of_measurement, + self.device_class, + value_template, + ["key"], + self.force_update, + self.resource_template, + self.json_attrs_path, + ) + + self.sensor.update() + assert {} == self.sensor.device_state_attributes + assert mock_logger.warning.called + assert mock_logger.debug.called + class TestRestData(unittest.TestCase): """Tests for RestData."""