evlli/hass-core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Generate nest images thumbnails from events (#44638)

Browse source 
* Capture nest still images from events

Use python google-nest-sdm API for fetching images.  Update home assistant
to use the google-nest-sdm API for fetching the image contents generated
by the server.  This uses the existing websession object for server fetches,
reducing the amount of new code and facilites unit testing using the existing
mechanism.

Simplify tests using the image fetch API rather than a snapshot API
This commit is contained in:
Allen Porter 2021-01-06 07:02:04 -08:00 • committed by GitHub
parent d3d66c2e27
commit 560e3811a3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 253 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
homeassistant/components/nest/camera_sdm.py
View file

@ -4,7 +4,11 @@ import datetime
import logging
from typing import Optional
from google_nest_sdm.camera_traits import CameraImageTrait, CameraLiveStreamTrait
from google_nest_sdm.camera_traits import (
CameraEventImageTrait,
CameraImageTrait,
CameraLiveStreamTrait,
)
from google_nest_sdm.device import Device
from google_nest_sdm.exceptions import GoogleNestException
from haffmpeg.tools import IMAGE_JPEG
@ -59,6 +63,9 @@ class NestCamera(Camera):
self._device_info = DeviceInfo(device)
self._stream = None
self._stream_refresh_unsub = None
# Cache of most recent event image
self._event_id = None
self._event_image_bytes = None
@property
def should_poll(self) -> bool:
@ -156,7 +163,40 @@ class NestCamera(Camera):
async def async_camera_image(self):
"""Return bytes of camera image."""
# Returns the snapshot of the last event for ~30 seconds after the event
active_event_image = await self._async_active_event_image()
if active_event_image:
return active_event_image
# Fetch still image from the live stream
stream_url = await self.stream_source()
if not stream_url:
return None
return await async_get_image(self.hass, stream_url, output_format=IMAGE_JPEG)
async def _async_active_event_image(self):
"""Return image from any active events happening."""
if CameraEventImageTrait.NAME not in self._device.traits:
return None
trait = self._device.active_event_trait
if not trait:
return None
# Reuse image bytes if they have already been fetched
event_id = trait.last_event.event_id
if self._event_id is not None and self._event_id == event_id:
return self._event_image_bytes
_LOGGER.info("Fetching URL for event_id %s", event_id)
try:
event_image = await trait.generate_active_event_image()
except GoogleNestException as err:
_LOGGER.debug("Unable to generate event image URL: %s", err)
return None
if not event_image:
return None
try:
image_bytes = await event_image.contents()
except GoogleNestException as err:
_LOGGER.debug("Unable to fetch event image: %s", err)
return None
self._event_id = event_id
self._event_image_bytes = image_bytes
return image_bytes

2
homeassistant/components/nest/manifest.json
View file

@ -6,7 +6,7 @@
"documentation": "https://www.home-assistant.io/integrations/nest",
"requirements": [
"python-nest==4.1.0",
"google-nest-sdm==0.2.5"
"google-nest-sdm==0.2.6"
],
"codeowners": [
"@awarecan",

2
requirements_all.txt
View file

@ -681,7 +681,7 @@ google-cloud-pubsub==2.1.0
google-cloud-texttospeech==0.4.0
# homeassistant.components.nest
google-nest-sdm==0.2.5
google-nest-sdm==0.2.6
# homeassistant.components.google_travel_time
googlemaps==2.5.1

2
requirements_test_all.txt
View file

@ -355,7 +355,7 @@ google-api-python-client==1.6.4
google-cloud-pubsub==2.1.0
# homeassistant.components.nest
google-nest-sdm==0.2.5
google-nest-sdm==0.2.6
# homeassistant.components.gree
greeclimate==0.10.3

331
tests/components/nest/camera_sdm_test.py
View file

@ -10,6 +10,7 @@ from unittest.mock import patch
import aiohttp
from google_nest_sdm.device import Device
from google_nest_sdm.event import EventMessage
import pytest
from homeassistant.components import camera
@ -36,9 +37,69 @@ DEVICE_TRAITS = {
"videoCodecs": ["H264"],
"audioCodecs": ["AAC"],
},
"sdm.devices.traits.CameraEventImage": {},
"sdm.devices.traits.CameraMotion": {},
}
DATETIME_FORMAT = "YY-MM-DDTHH:MM:SS"
DOMAIN = "nest"
MOTION_EVENT_ID = "FWWVQVUdGNUlTU2V4MGV2aTNXV..."
# Tests can assert that image bytes came from an event or was decoded
# from the live stream.
IMAGE_BYTES_FROM_EVENT = b"test url image bytes"
IMAGE_BYTES_FROM_STREAM = b"test stream image bytes"
TEST_IMAGE_URL = "https://domain/sdm_event_snapshot/dGTZwR3o4Y1..."
GENERATE_IMAGE_URL_RESPONSE = {
"results": {
"url": TEST_IMAGE_URL,
"token": "g.0.eventToken",
},
}
IMAGE_AUTHORIZATION_HEADERS = {"Authorization": "Basic g.0.eventToken"}
def make_motion_event(timestamp: datetime.datetime = None) -> EventMessage:
"""Create an EventMessage for a motion event."""
if not timestamp:
timestamp = utcnow()
return EventMessage(
{
"eventId": "some-event-id",
"timestamp": timestamp.isoformat(timespec="seconds"),
"resourceUpdate": {
"name": DEVICE_ID,
"events": {
"sdm.devices.events.CameraMotion.Motion": {
"eventSessionId": "CjY5Y3VKaTZwR3o4Y19YbTVfMF...",
"eventId": MOTION_EVENT_ID,
},
},
},
},
auth=None,
)
def make_stream_url_response(
expiration: datetime.datetime = None, token_num: int = 0
) -> aiohttp.web.Response:
"""Make response for the API that generates a streaming url."""
if not expiration:
# Default to an arbitrary time in the future
expiration = utcnow() + datetime.timedelta(seconds=100)
return aiohttp.web.json_response(
{
"results": {
"streamUrls": {
"rtspUrl": f"rtsp://some/url?auth=g.{token_num}.streamingToken"
},
"streamExtensionToken": f"g.{token_num}.extensionToken",
"streamToken": f"g.{token_num}.streamingToken",
"expiresAt": expiration.isoformat(timespec="seconds"),
},
}
)
async def async_setup_camera(hass, traits={}, auth=None):
@ -63,6 +124,19 @@ async def fire_alarm(hass, point_in_time):
await hass.async_block_till_done()
async def async_get_image(hass):
"""Get image from the camera, a wrapper around camera.async_get_image."""
# Note: this patches ImageFrame to simulate decoding an image from a live
# stream, however the test may not use it. Tests assert on the image
# contents to determine if the image came from the live stream or event.
with patch(
"homeassistant.components.ffmpeg.ImageFrame.get_image",
autopatch=True,
return_value=IMAGE_BYTES_FROM_STREAM,
):
return await camera.async_get_image(hass, "camera.my_camera")
async def test_no_devices(hass):
"""Test configuration that returns no devices."""
await async_setup_camera(hass)
@ -106,22 +180,7 @@ async def test_camera_device(hass):
async def test_camera_stream(hass, auth):
"""Test a basic camera and fetch its live stream."""
now = utcnow()
expiration = now + datetime.timedelta(seconds=100)
auth.responses = [
aiohttp.web.json_response(
{
"results": {
"streamUrls": {
"rtspUrl": "rtsp://some/url?auth=g.0.streamingToken"
},
"streamExtensionToken": "g.1.extensionToken",
"streamToken": "g.0.streamingToken",
"expiresAt": expiration.isoformat(timespec="seconds"),
},
}
)
]
auth.responses = [make_stream_url_response()]
await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
@ -132,14 +191,8 @@ async def test_camera_stream(hass, auth):
stream_source = await camera.async_get_stream_source(hass, "camera.my_camera")
assert stream_source == "rtsp://some/url?auth=g.0.streamingToken"
with patch(
"homeassistant.components.ffmpeg.ImageFrame.get_image",
autopatch=True,
return_value=b"image bytes",
):
image = await camera.async_get_image(hass, "camera.my_camera")
assert image.content == b"image bytes"
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
async def test_camera_stream_missing_trait(hass, auth):
@ -166,10 +219,9 @@ async def test_camera_stream_missing_trait(hass, auth):
stream_source = await camera.async_get_stream_source(hass, "camera.my_camera")
assert stream_source is None
# Currently on support getting the image from a live stream
# Unable to get an image from the live stream
with pytest.raises(HomeAssistantError):
image = await camera.async_get_image(hass, "camera.my_camera")
assert image is None
await async_get_image(hass)
async def test_refresh_expired_stream_token(hass, auth):
@ -180,38 +232,11 @@ async def test_refresh_expired_stream_token(hass, auth):
stream_3_expiration = now + datetime.timedelta(seconds=360)
auth.responses = [
# Stream URL #1
aiohttp.web.json_response(
{
"results": {
"streamUrls": {
"rtspUrl": "rtsp://some/url?auth=g.1.streamingToken"
},
"streamExtensionToken": "g.1.extensionToken",
"streamToken": "g.1.streamingToken",
"expiresAt": stream_1_expiration.isoformat(timespec="seconds"),
},
}
),
make_stream_url_response(stream_1_expiration, token_num=1),
# Stream URL #2
aiohttp.web.json_response(
{
"results": {
"streamExtensionToken": "g.2.extensionToken",
"streamToken": "g.2.streamingToken",
"expiresAt": stream_2_expiration.isoformat(timespec="seconds"),
},
}
),
make_stream_url_response(stream_2_expiration, token_num=2),
# Stream URL #3
aiohttp.web.json_response(
{
"results": {
"streamExtensionToken": "g.3.extensionToken",
"streamToken": "g.3.streamingToken",
"expiresAt": stream_3_expiration.isoformat(timespec="seconds"),
},
}
),
make_stream_url_response(stream_3_expiration, token_num=3),
]
await async_setup_camera(
hass,
@ -258,36 +283,10 @@ async def test_stream_response_already_expired(hass, auth):
stream_1_expiration = now + datetime.timedelta(seconds=-90)
stream_2_expiration = now + datetime.timedelta(seconds=+90)
auth.responses = [
aiohttp.web.json_response(
{
"results": {
"streamUrls": {
"rtspUrl": "rtsp://some/url?auth=g.1.streamingToken"
},
"streamExtensionToken": "g.1.extensionToken",
"streamToken": "g.1.streamingToken",
"expiresAt": stream_1_expiration.isoformat(timespec="seconds"),
},
}
),
aiohttp.web.json_response(
{
"results": {
"streamUrls": {
"rtspUrl": "rtsp://some/url?auth=g.2.streamingToken"
},
"streamExtensionToken": "g.2.extensionToken",
"streamToken": "g.2.streamingToken",
"expiresAt": stream_2_expiration.isoformat(timespec="seconds"),
},
}
),
make_stream_url_response(stream_1_expiration, token_num=1),
make_stream_url_response(stream_2_expiration, token_num=2),
]
await async_setup_camera(
hass,
DEVICE_TRAITS,
auth=auth,
)
await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
cam = hass.states.get("camera.my_camera")
@ -307,21 +306,8 @@ async def test_stream_response_already_expired(hass, auth):
async def test_camera_removed(hass, auth):
"""Test case where entities are removed and stream tokens expired."""
now = utcnow()
expiration = now + datetime.timedelta(seconds=100)
auth.responses = [
aiohttp.web.json_response(
{
"results": {
"streamUrls": {
"rtspUrl": "rtsp://some/url?auth=g.0.streamingToken"
},
"streamExtensionToken": "g.1.extensionToken",
"streamToken": "g.0.streamingToken",
"expiresAt": expiration.isoformat(timespec="seconds"),
},
}
),
make_stream_url_response(),
aiohttp.web.json_response({"results": {}}),
]
await async_setup_camera(
@ -349,39 +335,13 @@ async def test_refresh_expired_stream_failure(hass, auth):
stream_1_expiration = now + datetime.timedelta(seconds=90)
stream_2_expiration = now + datetime.timedelta(seconds=180)
auth.responses = [
aiohttp.web.json_response(
{
"results": {
"streamUrls": {
"rtspUrl": "rtsp://some/url?auth=g.1.streamingToken"
},
"streamExtensionToken": "g.1.extensionToken",
"streamToken": "g.1.streamingToken",
"expiresAt": stream_1_expiration.isoformat(timespec="seconds"),
},
}
),
make_stream_url_response(expiration=stream_1_expiration, token_num=1),
# Extending the stream fails with arbitrary error
aiohttp.web.Response(status=500),
# Next attempt to get a stream fetches a new url
aiohttp.web.json_response(
{
"results": {
"streamUrls": {
"rtspUrl": "rtsp://some/url?auth=g.2.streamingToken"
},
"streamExtensionToken": "g.2.extensionToken",
"streamToken": "g.2.streamingToken",
"expiresAt": stream_2_expiration.isoformat(timespec="seconds"),
},
}
),
make_stream_url_response(expiration=stream_2_expiration, token_num=2),
]
await async_setup_camera(
hass,
DEVICE_TRAITS,
auth=auth,
)
await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
cam = hass.states.get("camera.my_camera")
@ -399,3 +359,116 @@ async def test_refresh_expired_stream_failure(hass, auth):
# The stream is entirely refreshed
stream_source = await camera.async_get_stream_source(hass, "camera.my_camera")
assert stream_source == "rtsp://some/url?auth=g.2.streamingToken"
async def test_camera_image_from_last_event(hass, auth):
"""Test an image generated from an event."""
# The subscriber receives a message related to an image event. The camera
# holds on to the event message. When the test asks for a capera snapshot
# it exchanges the event id for an image url and fetches the image.
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
# Simulate a pubsub message received by the subscriber with a motion event.
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
auth.responses = [
# Fake response from API that returns url image
aiohttp.web.json_response(GENERATE_IMAGE_URL_RESPONSE),
# Fake response for the image content fetch
aiohttp.web.Response(body=IMAGE_BYTES_FROM_EVENT),
]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_EVENT
# Verify expected image fetch request was captured
assert auth.url == TEST_IMAGE_URL
assert auth.headers == IMAGE_AUTHORIZATION_HEADERS
# An additional fetch uses the cache and does not send another RPC
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_EVENT
# Verify expected image fetch request was captured
assert auth.url == TEST_IMAGE_URL
assert auth.headers == IMAGE_AUTHORIZATION_HEADERS
async def test_camera_image_from_event_not_supported(hass, auth):
"""Test fallback to stream image when event images are not supported."""
# Create a device that does not support the CameraEventImgae trait
traits = DEVICE_TRAITS.copy()
del traits["sdm.devices.traits.CameraEventImage"]
subscriber = await async_setup_camera(hass, traits, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
# Camera fetches a stream url since CameraEventImage is not supported
auth.responses = [make_stream_url_response()]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
async def test_generate_event_image_url_failure(hass, auth):
"""Test fallback to stream on failure to create an image url."""
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
auth.responses = [
# Fail to generate the image url
aiohttp.web.Response(status=500),
# Camera fetches a stream url as a fallback
make_stream_url_response(),
]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
async def test_fetch_event_image_failure(hass, auth):
"""Test fallback to a stream on image download failure."""
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
await subscriber.async_receive_event(make_motion_event())
await hass.async_block_till_done()
auth.responses = [
# Fake response from API that returns url image
aiohttp.web.json_response(GENERATE_IMAGE_URL_RESPONSE),
# Fail to download the image
aiohttp.web.Response(status=500),
# Camera fetches a stream url as a fallback
make_stream_url_response(),
]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM
async def test_event_image_expired(hass, auth):
"""Test fallback for an event event image that has expired."""
subscriber = await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
assert len(hass.states.async_all()) == 1
assert hass.states.get("camera.my_camera")
# Simulate a pubsub message has already expired
event_timestamp = utcnow() - datetime.timedelta(seconds=40)
await subscriber.async_receive_event(make_motion_event(event_timestamp))
await hass.async_block_till_done()
# Fallback to a stream url since the event message is expired.
auth.responses = [make_stream_url_response()]
image = await async_get_image(hass)
assert image.content == IMAGE_BYTES_FROM_STREAM

4
tests/components/nest/climate_sdm_test.py
View file

@ -933,14 +933,14 @@ async def test_thermostat_set_hvac_fan_only(hass, auth):
assert len(auth.captured_requests) == 2
(method, url, json) = auth.captured_requests.pop(0)
(method, url, json, headers) = auth.captured_requests.pop(0)
assert method == "post"
assert url == "some-device-id:executeCommand"
assert json == {
"command": "sdm.devices.commands.Fan.SetTimer",
"params": {"timerMode": "ON"},
}
(method, url, json) = auth.captured_requests.pop(0)
(method, url, json, headers) = auth.captured_requests.pop(0)
assert method == "post"
assert url == "some-device-id:executeCommand"
assert json == {

8
tests/components/nest/conftest.py
View file

@ -23,6 +23,7 @@ class FakeAuth(AbstractAuth):
self.method = None
self.url = None
self.json = None
self.headers = None
self.captured_requests = []
# Set up by fixture
self.client = None
@ -31,12 +32,13 @@ class FakeAuth(AbstractAuth):
"""Return a valid access token."""
return ""
async def request(self, method, url, json):
async def request(self, method, url, **kwargs):
"""Capure the request arguments for tests to assert on."""
self.method = method
self.url = url
self.json = json
self.captured_requests.append((method, url, json))
self.json = kwargs.get("json")
self.headers = kwargs.get("headers")
self.captured_requests.append((method, url, self.json, self.headers))
return await self.client.get("/")
async def response_handler(self, request):