Upgrade to aiohttp 3 (#12921)

* Upgrade aiohttp to 3.0.6

* Fix tests

* Fix aiohttp client stream test

* Lint

* Remove drain

homeassistant/components/api.py

@@ -131,8 +131,7 @@ class APIEventStream(HomeAssistantView):
                     msg = "data: {}\n\n".format(payload)
                     _LOGGER.debug('STREAM %s WRITING %s', id(stop_obj),
                                   msg.strip())
-                    response.write(msg.encode("UTF-8"))
+                    yield from response.write(msg.encode("UTF-8"))
-                    yield from response.drain()
                 except asyncio.TimeoutError:
                     yield from to_write.put(STREAM_PING_PAYLOAD)
 

homeassistant/components/camera/__init__.py

@@ -264,9 +264,9 @@ class Camera(Entity):
                                  'boundary=--frameboundary')
         yield from response.prepare(request)
 
-        def write(img_bytes):
+        async def write(img_bytes):
             """Write image to stream."""
-            response.write(bytes(
+            await response.write(bytes(
                 '--frameboundary\r\n'
                 'Content-Type: {}\r\n'
                 'Content-Length: {}\r\n\r\n'.format(
@@ -282,15 +282,14 @@ class Camera(Entity):
                     break
 
                 if img_bytes and img_bytes != last_image:
-                    write(img_bytes)
+                    yield from write(img_bytes)
 
                     # Chrome seems to always ignore first picture,
                     # print it twice.
                     if last_image is None:
-                        write(img_bytes)
+                        yield from write(img_bytes)
 
                     last_image = img_bytes
-                    yield from response.drain()
 
                 yield from asyncio.sleep(.5)
 

homeassistant/components/http/__init__.py

@@ -279,6 +279,10 @@ class HomeAssistantHTTP(object):
     @asyncio.coroutine
     def start(self):
         """Start the WSGI server."""
+        # We misunderstood the startup signal. You're not allowed to change
+        # anything during startup. Temp workaround.
+        # pylint: disable=protected-access
+        self.app._on_startup.freeze()
         yield from self.app.startup()
 
         if self.ssl_certificate:
@@ -298,10 +302,8 @@ class HomeAssistantHTTP(object):
         # Aiohttp freezes apps after start so that no changes can be made.
         # However in Home Assistant components can be discovered after boot.
         # This will now raise a RunTimeError.
-        # To work around this we now fake that we are frozen.
+        # To work around this we now prevent the router from getting frozen
-        # A more appropriate fix would be to create a new app and
+        self.app._router.freeze = lambda: None
-        # re-register all redirects, views, static paths.
-        self.app._frozen = True  # pylint: disable=protected-access
 
         self._handler = self.app.make_handler(loop=self.hass.loop)
 
@@ -312,10 +314,6 @@ class HomeAssistantHTTP(object):
             _LOGGER.error("Failed to create HTTP server at port %d: %s",
                           self.server_port, error)
 
-        # pylint: disable=protected-access
-        self.app._middlewares = tuple(self.app._prepare_middleware())
-        self.app._frozen = False
-
     @asyncio.coroutine
     def stop(self):
         """Stop the WSGI server."""

homeassistant/components/http/static.py

@@ -39,6 +39,7 @@ class CachingStaticResource(StaticResource):
             raise HTTPNotFound
 
 
+# pylint: disable=too-many-ancestors
 class CachingFileResponse(FileResponse):
     """FileSender class that caches output if not in dev mode."""
 

homeassistant/components/shopping_list.py

@@ -173,10 +173,9 @@ class UpdateShoppingListItemView(http.HomeAssistantView):
     url = '/api/shopping_list/item/{item_id}'
     name = "api:shopping_list:item:id"
 
-    @callback
+    async def post(self, request, item_id):
-    def post(self, request, item_id):
         """Update a shopping list item."""
-        data = yield from request.json()
+        data = await request.json()
 
         try:
             item = request.app['hass'].data[DOMAIN].async_update(item_id, data)

homeassistant/helpers/aiohttp_client.py

@@ -116,7 +116,7 @@ async def async_aiohttp_proxy_stream(hass, request, stream, content_type,
                 await response.write_eof()
                 break
 
-            response.write(data)
+            await response.write(data)
 
     except (asyncio.TimeoutError, aiohttp.ClientError):
         # Something went wrong fetching data, close connection gracefully

homeassistant/package_constraints.txt

@@ -5,10 +5,8 @@ pip>=8.0.3
 jinja2>=2.10
 voluptuous==0.11.1
 typing>=3,<4
-aiohttp==2.3.10
+aiohttp==3.0.6
-yarl==1.1.0
 async_timeout==2.0.0
-chardet==3.0.4
 astral==1.5
 certifi>=2017.4.17
 attrs==17.4.0

requirements_all.txt

@@ -6,10 +6,8 @@ pip>=8.0.3
 jinja2>=2.10
 voluptuous==0.11.1
 typing>=3,<4
-aiohttp==2.3.10
+aiohttp==3.0.6
-yarl==1.1.0
 async_timeout==2.0.0
-chardet==3.0.4
 astral==1.5
 certifi>=2017.4.17
 attrs==17.4.0

setup.py

@@ -50,10 +50,8 @@ REQUIRES = [
     'jinja2>=2.10',
     'voluptuous==0.11.1',
     'typing>=3,<4',
-    'aiohttp==2.3.10',   # If updated, check if yarl also needs an update!
+    'aiohttp==3.0.6',
-    'yarl==1.1.0',
     'async_timeout==2.0.0',
-    'chardet==3.0.4',
     'astral==1.5',
     'certifi>=2017.4.17',
     'attrs==17.4.0',

tests/components/test_websocket_api.py

@@ -23,7 +23,6 @@ def websocket_client(loop, hass, test_client):
 
     client = loop.run_until_complete(test_client(hass.http.app))
     ws = loop.run_until_complete(client.ws_connect(wapi.URL))
-
     auth_ok = loop.run_until_complete(ws.receive_json())
     assert auth_ok['type'] == wapi.TYPE_AUTH_OK
 
@@ -65,7 +64,7 @@ def mock_low_queue():
 @asyncio.coroutine
 def test_auth_via_msg(no_auth_websocket_client):
     """Test authenticating."""
-    no_auth_websocket_client.send_json({
+    yield from no_auth_websocket_client.send_json({
         'type': wapi.TYPE_AUTH,
         'api_password': API_PASSWORD
     })
@@ -80,7 +79,7 @@ def test_auth_via_msg_incorrect_pass(no_auth_websocket_client):
     """Test authenticating."""
     with patch('homeassistant.components.websocket_api.process_wrong_login',
                return_value=mock_coro()) as mock_process_wrong_login:
-        no_auth_websocket_client.send_json({
+        yield from no_auth_websocket_client.send_json({
             'type': wapi.TYPE_AUTH,
             'api_password': API_PASSWORD + 'wrong'
         })
@@ -95,7 +94,7 @@ def test_auth_via_msg_incorrect_pass(no_auth_websocket_client):
 @asyncio.coroutine
 def test_pre_auth_only_auth_allowed(no_auth_websocket_client):
     """Verify that before authentication, only auth messages are allowed."""
-    no_auth_websocket_client.send_json({
+    yield from no_auth_websocket_client.send_json({
         'type': wapi.TYPE_CALL_SERVICE,
         'domain': 'domain_test',
         'service': 'test_service',
@@ -113,7 +112,7 @@ def test_pre_auth_only_auth_allowed(no_auth_websocket_client):
 @asyncio.coroutine
 def test_invalid_message_format(websocket_client):
     """Test sending invalid JSON."""
-    websocket_client.send_json({'type': 5})
+    yield from websocket_client.send_json({'type': 5})
 
     msg = yield from websocket_client.receive_json()
 
@@ -126,7 +125,7 @@ def test_invalid_message_format(websocket_client):
 @asyncio.coroutine
 def test_invalid_json(websocket_client):
     """Test sending invalid JSON."""
-    websocket_client.send_str('this is not JSON')
+    yield from websocket_client.send_str('this is not JSON')
 
     msg = yield from websocket_client.receive()
 
@@ -155,7 +154,7 @@ def test_call_service(hass, websocket_client):
 
     hass.services.async_register('domain_test', 'test_service', service_call)
 
-    websocket_client.send_json({
+    yield from websocket_client.send_json({
         'id': 5,
         'type': wapi.TYPE_CALL_SERVICE,
         'domain': 'domain_test',
@@ -183,7 +182,7 @@ def test_subscribe_unsubscribe_events(hass, websocket_client):
     """Test subscribe/unsubscribe events command."""
     init_count = sum(hass.bus.async_listeners().values())
 
-    websocket_client.send_json({
+    yield from websocket_client.send_json({
         'id': 5,
         'type': wapi.TYPE_SUBSCRIBE_EVENTS,
         'event_type': 'test_event'
@@ -212,7 +211,7 @@ def test_subscribe_unsubscribe_events(hass, websocket_client):
     assert event['data'] == {'hello': 'world'}
     assert event['origin'] == 'LOCAL'
 
-    websocket_client.send_json({
+    yield from websocket_client.send_json({
         'id': 6,
         'type': wapi.TYPE_UNSUBSCRIBE_EVENTS,
         'subscription': 5
@@ -233,7 +232,7 @@ def test_get_states(hass, websocket_client):
     hass.states.async_set('greeting.hello', 'world')
     hass.states.async_set('greeting.bye', 'universe')
 
-    websocket_client.send_json({
+    yield from websocket_client.send_json({
         'id': 5,
         'type': wapi.TYPE_GET_STATES,
     })
@@ -256,7 +255,7 @@ def test_get_states(hass, websocket_client):
 @asyncio.coroutine
 def test_get_services(hass, websocket_client):
     """Test get_services command."""
-    websocket_client.send_json({
+    yield from websocket_client.send_json({
         'id': 5,
         'type': wapi.TYPE_GET_SERVICES,
     })
@@ -271,7 +270,7 @@ def test_get_services(hass, websocket_client):
 @asyncio.coroutine
 def test_get_config(hass, websocket_client):
     """Test get_config command."""
-    websocket_client.send_json({
+    yield from websocket_client.send_json({
         'id': 5,
         'type': wapi.TYPE_GET_CONFIG,
     })
@@ -296,7 +295,7 @@ def test_get_panels(hass, websocket_client):
     yield from hass.components.frontend.async_register_built_in_panel(
         'map', 'Map', 'mdi:account-location')
     hass.data[frontend.DATA_JS_VERSION] = 'es5'
-    websocket_client.send_json({
+    yield from websocket_client.send_json({
         'id': 5,
         'type': wapi.TYPE_GET_PANELS,
     })
@@ -318,7 +317,7 @@ def test_get_panels(hass, websocket_client):
 @asyncio.coroutine
 def test_ping(websocket_client):
     """Test get_panels command."""
-    websocket_client.send_json({
+    yield from websocket_client.send_json({
         'id': 5,
         'type': wapi.TYPE_PING,
     })
@@ -332,7 +331,7 @@ def test_ping(websocket_client):
 def test_pending_msg_overflow(hass, mock_low_queue, websocket_client):
     """Test get_panels command."""
     for idx in range(10):
-        websocket_client.send_json({
+        yield from websocket_client.send_json({
             'id': idx + 1,
             'type': wapi.TYPE_PING,
         })

tests/helpers/test_aiohttp_client.py

@@ -3,6 +3,7 @@ import asyncio
 import unittest
 
 import aiohttp
+import pytest
 
 from homeassistant.core import EVENT_HOMEASSISTANT_CLOSE
 from homeassistant.setup import async_setup_component
@@ -12,6 +13,19 @@ from homeassistant.util.async import run_callback_threadsafe
 from tests.common import get_test_home_assistant
 
 
+@pytest.fixture
+def camera_client(hass, test_client):
+    """Fixture to fetch camera streams."""
+    assert hass.loop.run_until_complete(async_setup_component(hass, 'camera', {
+        'camera': {
+            'name': 'config_test',
+            'platform': 'mjpeg',
+            'mjpeg_url': 'http://example.com/mjpeg_stream',
+        }}))
+
+    yield hass.loop.run_until_complete(test_client(hass.http.app))
+
+
 class TestHelpersAiohttpClient(unittest.TestCase):
     """Test homeassistant.helpers.aiohttp_client module."""
 
@@ -119,41 +133,38 @@ class TestHelpersAiohttpClient(unittest.TestCase):
 
 
 @asyncio.coroutine
-def test_async_aiohttp_proxy_stream(aioclient_mock, hass, test_client):
+def test_async_aiohttp_proxy_stream(aioclient_mock, camera_client):
     """Test that it fetches the given url."""
     aioclient_mock.get('http://example.com/mjpeg_stream', content=[
         b'Frame1', b'Frame2', b'Frame3'
     ])
 
-    result = yield from async_setup_component(hass, 'camera', {
+    resp = yield from camera_client.get(
-        'camera': {
+        '/api/camera_proxy_stream/camera.config_test')
-            'name': 'config_test',
-            'platform': 'mjpeg',
-            'mjpeg_url': 'http://example.com/mjpeg_stream',
-        }})
-    assert result, 'Failed to setup camera'
-
-    client = yield from test_client(hass.http.app)
-
-    resp = yield from client.get('/api/camera_proxy_stream/camera.config_test')
 
     assert resp.status == 200
     assert aioclient_mock.call_count == 1
     body = yield from resp.text()
     assert body == 'Frame3Frame2Frame1'
 
-    aioclient_mock.clear_requests()
-    aioclient_mock.get(
-        'http://example.com/mjpeg_stream', exc=asyncio.TimeoutError(),
-        content=[b'Frame1', b'Frame2', b'Frame3'])
 
-    resp = yield from client.get('/api/camera_proxy_stream/camera.config_test')
+@asyncio.coroutine
+def test_async_aiohttp_proxy_stream_timeout(aioclient_mock, camera_client):
+    """Test that it fetches the given url."""
+    aioclient_mock.get(
+        'http://example.com/mjpeg_stream', exc=asyncio.TimeoutError())
+
+    resp = yield from camera_client.get(
+        '/api/camera_proxy_stream/camera.config_test')
     assert resp.status == 504
 
-    aioclient_mock.clear_requests()
-    aioclient_mock.get(
-        'http://example.com/mjpeg_stream', exc=aiohttp.ClientError(),
-        content=[b'Frame1', b'Frame2', b'Frame3'])
 
-    resp = yield from client.get('/api/camera_proxy_stream/camera.config_test')
+@asyncio.coroutine
+def test_async_aiohttp_proxy_stream_client_err(aioclient_mock, camera_client):
+    """Test that it fetches the given url."""
+    aioclient_mock.get(
+        'http://example.com/mjpeg_stream', exc=aiohttp.ClientError())
+
+    resp = yield from camera_client.get(
+        '/api/camera_proxy_stream/camera.config_test')
     assert resp.status == 502