diff --git a/homeassistant/components/nest/__init__.py b/homeassistant/components/nest/__init__.py
index ea2bd549c4b..c70184c357b 100644
--- a/homeassistant/components/nest/__init__.py
+++ b/homeassistant/components/nest/__init__.py
@@ -6,6 +6,7 @@ import logging
 import threading
 
 from google_nest_sdm.event import EventCallback, EventMessage
+from google_nest_sdm.exceptions import GoogleNestException
 from google_nest_sdm.google_nest_subscriber import GoogleNestSubscriber
 from nest import Nest
 from nest.nest import APIError, AuthorizationError
@@ -25,6 +26,7 @@ from homeassistant.const import (
     EVENT_HOMEASSISTANT_STOP,
 )
 from homeassistant.core import HomeAssistant, callback
+from homeassistant.exceptions import ConfigEntryNotReady
 from homeassistant.helpers import (
     aiohttp_client,
     config_entry_oauth2_flow,
@@ -208,7 +210,21 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry):
         auth, config[CONF_PROJECT_ID], config[CONF_SUBSCRIBER_ID]
     )
     subscriber.set_update_callback(SignalUpdateCallback(hass))
-    asyncio.create_task(subscriber.start_async())
+
+    try:
+        await subscriber.start_async()
+    except GoogleNestException as err:
+        _LOGGER.error("Subscriber error: %s", err)
+        subscriber.stop_async()
+        raise ConfigEntryNotReady from err
+
+    try:
+        await subscriber.async_get_device_manager()
+    except GoogleNestException as err:
+        _LOGGER.error("Device Manager error: %s", err)
+        subscriber.stop_async()
+        raise ConfigEntryNotReady from err
+
     hass.data[DOMAIN][entry.entry_id] = subscriber
 
     for component in PLATFORMS:
diff --git a/homeassistant/components/nest/camera_sdm.py b/homeassistant/components/nest/camera_sdm.py
index a8c5c86c4e8..bd06fb0bd8d 100644
--- a/homeassistant/components/nest/camera_sdm.py
+++ b/homeassistant/components/nest/camera_sdm.py
@@ -4,14 +4,15 @@ import datetime
 import logging
 from typing import Optional
 
-from aiohttp.client_exceptions import ClientError
 from google_nest_sdm.camera_traits import CameraImageTrait, CameraLiveStreamTrait
 from google_nest_sdm.device import Device
+from google_nest_sdm.exceptions import GoogleNestException
 from haffmpeg.tools import IMAGE_JPEG
 
 from homeassistant.components.camera import SUPPORT_STREAM, Camera
 from homeassistant.components.ffmpeg import async_get_image
 from homeassistant.config_entries import ConfigEntry
+from homeassistant.exceptions import PlatformNotReady
 from homeassistant.helpers.dispatcher import async_dispatcher_connect
 from homeassistant.helpers.event import async_track_point_in_utc_time
 from homeassistant.helpers.typing import HomeAssistantType
@@ -32,7 +33,10 @@ async def async_setup_sdm_entry(
     """Set up the cameras."""
 
     subscriber = hass.data[DOMAIN][entry.entry_id]
-    device_manager = await subscriber.async_get_device_manager()
+    try:
+        device_manager = await subscriber.async_get_device_manager()
+    except GoogleNestException as err:
+        raise PlatformNotReady from err
 
     # Fetch initial data so we have data when entities subscribe.
 
@@ -130,7 +134,7 @@ class NestCamera(Camera):
         self._stream_refresh_unsub = None
         try:
             self._stream = await self._stream.extend_rtsp_stream()
-        except ClientError as err:
+        except GoogleNestException as err:
             _LOGGER.debug("Failed to extend stream: %s", err)
             # Next attempt to catch a url will get a new one
             self._stream = None
diff --git a/homeassistant/components/nest/climate_sdm.py b/homeassistant/components/nest/climate_sdm.py
index 57598e36ec9..f341b76c404 100644
--- a/homeassistant/components/nest/climate_sdm.py
+++ b/homeassistant/components/nest/climate_sdm.py
@@ -4,6 +4,7 @@ from typing import Optional
 
 from google_nest_sdm.device import Device
 from google_nest_sdm.device_traits import FanTrait, TemperatureTrait
+from google_nest_sdm.exceptions import GoogleNestException
 from google_nest_sdm.thermostat_traits import (
     ThermostatEcoTrait,
     ThermostatHvacTrait,
@@ -34,6 +35,7 @@ from homeassistant.components.climate.const import (
 )
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.const import ATTR_TEMPERATURE, TEMP_CELSIUS
+from homeassistant.exceptions import PlatformNotReady
 from homeassistant.helpers.dispatcher import async_dispatcher_connect
 from homeassistant.helpers.typing import HomeAssistantType
 
@@ -80,7 +82,10 @@ async def async_setup_sdm_entry(
     """Set up the client entities."""
 
     subscriber = hass.data[DOMAIN][entry.entry_id]
-    device_manager = await subscriber.async_get_device_manager()
+    try:
+        device_manager = await subscriber.async_get_device_manager()
+    except GoogleNestException as err:
+        raise PlatformNotReady from err
 
     entities = []
     for device in device_manager.devices.values():
diff --git a/homeassistant/components/nest/manifest.json b/homeassistant/components/nest/manifest.json
index 9e8a48fa95f..b994fcfbfce 100644
--- a/homeassistant/components/nest/manifest.json
+++ b/homeassistant/components/nest/manifest.json
@@ -6,7 +6,7 @@
   "documentation": "https://www.home-assistant.io/integrations/nest",
   "requirements": [
       "python-nest==4.1.0",
-      "google-nest-sdm==0.1.14"
+      "google-nest-sdm==0.1.15"
   ],
   "codeowners": [
       "@awarecan",
diff --git a/homeassistant/components/nest/sensor_sdm.py b/homeassistant/components/nest/sensor_sdm.py
index 68c33529831..a0a28756ac2 100644
--- a/homeassistant/components/nest/sensor_sdm.py
+++ b/homeassistant/components/nest/sensor_sdm.py
@@ -1,9 +1,11 @@
 """Support for Google Nest SDM sensors."""
 
+import logging
 from typing import Optional
 
 from google_nest_sdm.device import Device
 from google_nest_sdm.device_traits import HumidityTrait, TemperatureTrait
+from google_nest_sdm.exceptions import GoogleNestException
 
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.const import (
@@ -12,6 +14,7 @@ from homeassistant.const import (
     PERCENTAGE,
     TEMP_CELSIUS,
 )
+from homeassistant.exceptions import PlatformNotReady
 from homeassistant.helpers.dispatcher import async_dispatcher_connect
 from homeassistant.helpers.entity import Entity
 from homeassistant.helpers.typing import HomeAssistantType
@@ -19,6 +22,9 @@ from homeassistant.helpers.typing import HomeAssistantType
 from .const import DOMAIN, SIGNAL_NEST_UPDATE
 from .device_info import DeviceInfo
 
+_LOGGER = logging.getLogger(__name__)
+
+
 DEVICE_TYPE_MAP = {
     "sdm.devices.types.CAMERA": "Camera",
     "sdm.devices.types.DISPLAY": "Display",
@@ -33,7 +39,11 @@ async def async_setup_sdm_entry(
     """Set up the sensors."""
 
     subscriber = hass.data[DOMAIN][entry.entry_id]
-    device_manager = await subscriber.async_get_device_manager()
+    try:
+        device_manager = await subscriber.async_get_device_manager()
+    except GoogleNestException as err:
+        _LOGGER.warning("Failed to get devices: %s", err)
+        raise PlatformNotReady from err
 
     # Fetch initial data so we have data when entities subscribe.
 
diff --git a/requirements_all.txt b/requirements_all.txt
index d4a35de3e97..b1d1fddbe33 100644
--- a/requirements_all.txt
+++ b/requirements_all.txt
@@ -687,7 +687,7 @@ google-cloud-pubsub==2.1.0
 google-cloud-texttospeech==0.4.0
 
 # homeassistant.components.nest
-google-nest-sdm==0.1.14
+google-nest-sdm==0.1.15
 
 # homeassistant.components.google_travel_time
 googlemaps==2.5.1
diff --git a/requirements_test_all.txt b/requirements_test_all.txt
index de236d4f0fd..53f73f346d2 100644
--- a/requirements_test_all.txt
+++ b/requirements_test_all.txt
@@ -358,7 +358,7 @@ google-api-python-client==1.6.4
 google-cloud-pubsub==2.1.0
 
 # homeassistant.components.nest
-google-nest-sdm==0.1.14
+google-nest-sdm==0.1.15
 
 # homeassistant.components.gree
 greeclimate==0.10.3
diff --git a/tests/components/nest/camera_sdm_test.py b/tests/components/nest/camera_sdm_test.py
index e3397129ec9..4a018305bcf 100644
--- a/tests/components/nest/camera_sdm_test.py
+++ b/tests/components/nest/camera_sdm_test.py
@@ -6,10 +6,8 @@ pubsub subscriber.
 """
 
 import datetime
-from typing import List
 
-from aiohttp.client_exceptions import ClientConnectionError
-from google_nest_sdm.auth import AbstractAuth
+import aiohttp
 from google_nest_sdm.device import Device
 
 from homeassistant.components import camera
@@ -41,47 +39,6 @@ DATETIME_FORMAT = "YY-MM-DDTHH:MM:SS"
 DOMAIN = "nest"
 
 
-class FakeResponse:
-    """A fake web response used for returning results of commands."""
-
-    def __init__(self, json=None, error=None):
-        """Initialize the FakeResponse."""
-        self._json = json
-        self._error = error
-
-    def raise_for_status(self):
-        """Mimics a successful response status."""
-        if self._error:
-            raise self._error
-        pass
-
-    async def json(self):
-        """Return a dict with the response."""
-        assert self._json
-        return self._json
-
-
-class FakeAuth(AbstractAuth):
-    """Fake authentication object that returns fake responses."""
-
-    def __init__(self, responses: List[FakeResponse]):
-        """Initialize the FakeAuth."""
-        super().__init__(None, "")
-        self._responses = responses
-
-    async def async_get_access_token(self):
-        """Return a fake access token."""
-        return "some-token"
-
-    async def creds(self):
-        """Return a fake creds."""
-        return None
-
-    async def request(self, method: str, url: str, **kwargs):
-        """Pass through the FakeResponse."""
-        return self._responses.pop(0)
-
-
 async def async_setup_camera(hass, traits={}, auth=None):
     """Set up the platform and prerequisites."""
     devices = {}
@@ -145,21 +102,25 @@ async def test_camera_device(hass):
     assert device.identifiers == {("nest", DEVICE_ID)}
 
 
-async def test_camera_stream(hass, aiohttp_client):
+async def test_camera_stream(hass, auth):
     """Test a basic camera and fetch its live stream."""
     now = utcnow()
     expiration = now + datetime.timedelta(seconds=100)
-    response = FakeResponse(
-        {
-            "results": {
-                "streamUrls": {"rtspUrl": "rtsp://some/url?auth=g.0.streamingToken"},
-                "streamExtensionToken": "g.1.extensionToken",
-                "streamToken": "g.0.streamingToken",
-                "expiresAt": expiration.isoformat(timespec="seconds"),
-            },
-        }
-    )
-    await async_setup_camera(hass, DEVICE_TRAITS, auth=FakeAuth([response]))
+    auth.responses = [
+        aiohttp.web.json_response(
+            {
+                "results": {
+                    "streamUrls": {
+                        "rtspUrl": "rtsp://some/url?auth=g.0.streamingToken"
+                    },
+                    "streamExtensionToken": "g.1.extensionToken",
+                    "streamToken": "g.0.streamingToken",
+                    "expiresAt": expiration.isoformat(timespec="seconds"),
+                },
+            }
+        )
+    ]
+    await async_setup_camera(hass, DEVICE_TRAITS, auth=auth)
 
     assert len(hass.states.async_all()) == 1
     cam = hass.states.get("camera.my_camera")
@@ -179,15 +140,15 @@ async def test_camera_stream(hass, aiohttp_client):
     assert image.content == b"image bytes"
 
 
-async def test_refresh_expired_stream_token(hass, aiohttp_client):
+async def test_refresh_expired_stream_token(hass, auth):
     """Test a camera stream expiration and refresh."""
     now = utcnow()
     stream_1_expiration = now + datetime.timedelta(seconds=90)
     stream_2_expiration = now + datetime.timedelta(seconds=180)
     stream_3_expiration = now + datetime.timedelta(seconds=360)
-    responses = [
+    auth.responses = [
         # Stream URL #1
-        FakeResponse(
+        aiohttp.web.json_response(
             {
                 "results": {
                     "streamUrls": {
@@ -200,7 +161,7 @@ async def test_refresh_expired_stream_token(hass, aiohttp_client):
             }
         ),
         # Stream URL #2
-        FakeResponse(
+        aiohttp.web.json_response(
             {
                 "results": {
                     "streamExtensionToken": "g.2.extensionToken",
@@ -210,7 +171,7 @@ async def test_refresh_expired_stream_token(hass, aiohttp_client):
             }
         ),
         # Stream URL #3
-        FakeResponse(
+        aiohttp.web.json_response(
             {
                 "results": {
                     "streamExtensionToken": "g.3.extensionToken",
@@ -223,7 +184,7 @@ async def test_refresh_expired_stream_token(hass, aiohttp_client):
     await async_setup_camera(
         hass,
         DEVICE_TRAITS,
-        auth=FakeAuth(responses),
+        auth=auth,
     )
 
     assert len(hass.states.async_all()) == 1
@@ -259,12 +220,12 @@ async def test_refresh_expired_stream_token(hass, aiohttp_client):
     assert stream_source == "rtsp://some/url?auth=g.3.streamingToken"
 
 
-async def test_camera_removed(hass, aiohttp_client):
+async def test_camera_removed(hass, auth):
     """Test case where entities are removed and stream tokens expired."""
     now = utcnow()
     expiration = now + datetime.timedelta(seconds=100)
-    responses = [
-        FakeResponse(
+    auth.responses = [
+        aiohttp.web.json_response(
             {
                 "results": {
                     "streamUrls": {
@@ -276,12 +237,12 @@ async def test_camera_removed(hass, aiohttp_client):
                 },
             }
         ),
-        FakeResponse({"results": {}}),
+        aiohttp.web.json_response({"results": {}}),
     ]
     await async_setup_camera(
         hass,
         DEVICE_TRAITS,
-        auth=FakeAuth(responses),
+        auth=auth,
     )
 
     assert len(hass.states.async_all()) == 1
@@ -297,13 +258,13 @@ async def test_camera_removed(hass, aiohttp_client):
     assert len(hass.states.async_all()) == 0
 
 
-async def test_refresh_expired_stream_failure(hass, aiohttp_client):
+async def test_refresh_expired_stream_failure(hass, auth):
     """Tests a failure when refreshing the stream."""
     now = utcnow()
     stream_1_expiration = now + datetime.timedelta(seconds=90)
     stream_2_expiration = now + datetime.timedelta(seconds=180)
-    responses = [
-        FakeResponse(
+    auth.responses = [
+        aiohttp.web.json_response(
             {
                 "results": {
                     "streamUrls": {
@@ -316,9 +277,9 @@ async def test_refresh_expired_stream_failure(hass, aiohttp_client):
             }
         ),
         # Extending the stream fails with arbitrary error
-        FakeResponse(error=ClientConnectionError()),
+        aiohttp.web.Response(status=500),
         # Next attempt to get a stream fetches a new url
-        FakeResponse(
+        aiohttp.web.json_response(
             {
                 "results": {
                     "streamUrls": {
@@ -334,7 +295,7 @@ async def test_refresh_expired_stream_failure(hass, aiohttp_client):
     await async_setup_camera(
         hass,
         DEVICE_TRAITS,
-        auth=FakeAuth(responses),
+        auth=auth,
     )
 
     assert len(hass.states.async_all()) == 1
diff --git a/tests/components/nest/climate_sdm_test.py b/tests/components/nest/climate_sdm_test.py
index 48efd32d859..4c7ec4c0163 100644
--- a/tests/components/nest/climate_sdm_test.py
+++ b/tests/components/nest/climate_sdm_test.py
@@ -364,25 +364,8 @@ async def test_thermostat_eco_heat_only(hass):
     assert thermostat.attributes[ATTR_PRESET_MODES] == [PRESET_ECO, PRESET_NONE]
 
 
-class FakeAuth:
-    """A fake implementation of the auth class that records requests."""
-
-    def __init__(self):
-        """Initialize FakeAuth."""
-        self.method = None
-        self.url = None
-        self.json = None
-
-    async def request(self, method, url, json):
-        """Capure the request arguments for tests to assert on."""
-        self.method = method
-        self.url = url
-        self.json = json
-
-
-async def test_thermostat_set_hvac_mode(hass):
+async def test_thermostat_set_hvac_mode(hass, auth):
     """Test a thermostat changing hvac modes."""
-    auth = FakeAuth()
     subscriber = await setup_climate(
         hass,
         {
@@ -467,9 +450,8 @@ async def test_thermostat_set_hvac_mode(hass):
     assert thermostat.attributes[ATTR_HVAC_ACTION] == CURRENT_HVAC_HEAT
 
 
-async def test_thermostat_set_eco_preset(hass):
+async def test_thermostat_set_eco_preset(hass, auth):
     """Test a thermostat put into eco mode."""
-    auth = FakeAuth()
     subscriber = await setup_climate(
         hass,
         {
@@ -553,9 +535,8 @@ async def test_thermostat_set_eco_preset(hass):
     }
 
 
-async def test_thermostat_set_cool(hass):
+async def test_thermostat_set_cool(hass, auth):
     """Test a thermostat in cool mode with a temperature change."""
-    auth = FakeAuth()
     await setup_climate(
         hass,
         {
@@ -587,9 +568,8 @@ async def test_thermostat_set_cool(hass):
     }
 
 
-async def test_thermostat_set_heat(hass):
+async def test_thermostat_set_heat(hass, auth):
     """Test a thermostat heating mode with a temperature change."""
-    auth = FakeAuth()
     await setup_climate(
         hass,
         {
@@ -621,9 +601,8 @@ async def test_thermostat_set_heat(hass):
     }
 
 
-async def test_thermostat_set_heat_cool(hass):
+async def test_thermostat_set_heat_cool(hass, auth):
     """Test a thermostat in heatcool mode with a temperature change."""
-    auth = FakeAuth()
     await setup_climate(
         hass,
         {
@@ -732,9 +711,8 @@ async def test_thermostat_fan_on(hass):
     assert thermostat.attributes[ATTR_FAN_MODES] == [FAN_ON, FAN_OFF]
 
 
-async def test_thermostat_set_fan(hass):
+async def test_thermostat_set_fan(hass, auth):
     """Test a thermostat enabling the fan."""
-    auth = FakeAuth()
     await setup_climate(
         hass,
         {
@@ -805,9 +783,8 @@ async def test_thermostat_fan_empty(hass):
     assert ATTR_FAN_MODES not in thermostat.attributes
 
 
-async def test_thermostat_target_temp(hass):
+async def test_thermostat_target_temp(hass, auth):
     """Test a thermostat changing hvac modes and affected on target temps."""
-    auth = FakeAuth()
     subscriber = await setup_climate(
         hass,
         {
diff --git a/tests/components/nest/conftest.py b/tests/components/nest/conftest.py
new file mode 100644
index 00000000000..7e183ab9c82
--- /dev/null
+++ b/tests/components/nest/conftest.py
@@ -0,0 +1,57 @@
+"""Common libraries for test setup."""
+
+import aiohttp
+from google_nest_sdm.auth import AbstractAuth
+import pytest
+
+
+class FakeAuth(AbstractAuth):
+    """A fake implementation of the auth class that records requests.
+
+    This class captures the outgoing requests, and can also be used by
+    tests to set up fake responses.  This class is registered as a response
+    handler for a fake aiohttp_server and can simulate successes or failures
+    from the API.
+    """
+
+    # Tests can set fake responses here.
+    responses = []
+    # The last request is recorded here.
+    method = None
+    url = None
+    json = None
+
+    # Set up by fixture
+    client = None
+
+    def __init__(self):
+        """Initialize FakeAuth."""
+        super().__init__(None, None)
+
+    async def async_get_access_token(self) -> str:
+        """Return a valid access token."""
+        return ""
+
+    async def request(self, method, url, json):
+        """Capure the request arguments for tests to assert on."""
+        self.method = method
+        self.url = url
+        self.json = json
+        return await self.client.get("/")
+
+    async def response_handler(self, request):
+        """Handle fake responess for aiohttp_server."""
+        if len(self.responses) > 0:
+            return self.responses.pop(0)
+        return aiohttp.web.json_response()
+
+
+@pytest.fixture
+async def auth(aiohttp_client):
+    """Fixture for an AbstractAuth."""
+    auth = FakeAuth()
+    app = aiohttp.web.Application()
+    app.router.add_get("/", auth.response_handler)
+    app.router.add_post("/", auth.response_handler)
+    auth.client = await aiohttp_client(app)
+    return auth