Macvendor (#4468)

* Add MAC vendor lookup for device_tracker.

* Test vendor mac lookup and fix device attribute.

* Generate requirements.

* Style.

* Use hyphen instead of underscore to satisfy 'idna'.

https://github.com/kjd/idna/issues/17

* Resort imports.

* Refactor macvendor to use macvendors.com API instead of netaddr library.

* Test vendor lookup using macvendors.com api.

* Remove debugging.

* Correct description.

* No longer needed.

* Device tracker is now an async component. Fix ddwrt tests.

* Fix linting.

* Add test case for error conditions.

* There is no reason to retry failes vendor loopups as they won't be saved to the file anyways at that point.

* Sorry, bad assumption, this only made things worse.

* Wait for async parts during setup component to complete before asserting results.

* Fix linting.

* Is generated when running 'coverage html'.

* Undo isort.

* Make aioclient_mock exception more generic.

* Only lookup mac vendor string with adding new device to known_devices.yaml.

* Undo isort.

* Revert unneeded change.

* Adjust to use new websession pattern.

* Always make sure to cleanup response.

* Use correct function to release response.

* Fix tests.

.gitignore

@@ -62,6 +62,7 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
+htmlcov/
 
 # Translations
 *.mo

homeassistant/components/device_tracker/__init__.py

@@ -10,6 +10,8 @@ import logging
 import os
 from typing import Any, Sequence, Callable
 
+import aiohttp
+import async_timeout
 import voluptuous as vol
 
 from homeassistant.bootstrap import (
@@ -19,6 +21,7 @@ from homeassistant.components import group, zone
 from homeassistant.components.discovery import SERVICE_NETGEAR
 from homeassistant.config import load_yaml_config_file
 from homeassistant.exceptions import HomeAssistantError
+from homeassistant.helpers.aiohttp_client import async_get_clientsession
 from homeassistant.helpers import config_per_platform, discovery
 from homeassistant.helpers.entity import Entity
 from homeassistant.helpers.typing import GPSType, ConfigType, HomeAssistantType
@@ -278,6 +281,9 @@ class DeviceTracker(object):
             yield from self.group.async_update_tracked_entity_ids(
                 list(self.group.tracking) + [device.entity_id])
 
+        # lookup mac vendor string to be stored in config
+        device.set_vendor_for_mac()
+
         # update known_devices.yaml
         self.hass.async_add_job(
             self.async_update_config(self.hass.config.path(YAML_DEVICES),
@@ -328,6 +334,7 @@ class Device(Entity):
     last_seen = None  # type: dt_util.dt.datetime
     battery = None  # type: str
     attributes = None  # type: dict
+    vendor = None  # type: str
 
     # Track if the last update of this device was HOME.
     last_update_home = False
@@ -336,7 +343,7 @@ class Device(Entity):
     def __init__(self, hass: HomeAssistantType, consider_home: timedelta,
                  track: bool, dev_id: str, mac: str, name: str=None,
                  picture: str=None, gravatar: str=None,
-                 hide_if_away: bool=False) -> None:
+                 hide_if_away: bool=False, vendor: str=None) -> None:
         """Initialize a device."""
         self.hass = hass
         self.entity_id = ENTITY_ID_FORMAT.format(dev_id)
@@ -362,6 +369,7 @@ class Device(Entity):
             self.config_picture = picture
 
         self.away_hide = hide_if_away
+        self.vendor = vendor
 
     @property
     def name(self):
@@ -460,6 +468,53 @@ class Device(Entity):
             self._state = STATE_HOME
             self.last_update_home = True
 
+    @asyncio.coroutine
+    def set_vendor_for_mac(self):
+        """Set vendor string using api.macvendors.com."""
+        self.vendor = yield from self.get_vendor_for_mac()
+
+    @asyncio.coroutine
+    def get_vendor_for_mac(self):
+        """Try to find the vendor string for a given MAC address."""
+        # can't continue without a mac
+        if not self.mac:
+            return None
+
+        # prevent lookup of invalid macs
+        if not len(self.mac.split(':')) == 6:
+            return 'unknown'
+
+        # we only need the first 3 bytes of the mac for a lookup
+        # this improves somewhat on privacy
+        oui_bytes = self.mac.split(':')[0:3]
+        # bytes like 00 get truncates to 0, API needs full bytes
+        oui = '{:02x}:{:02x}:{:02x}'.format(*[int(b, 16) for b in oui_bytes])
+        url = 'http://api.macvendors.com/' + oui
+        resp = None
+        try:
+            websession = async_get_clientsession(self.hass)
+
+            with async_timeout.timeout(5, loop=self.hass.loop):
+                resp = yield from websession.get(url)
+            # mac vendor found, response is the string
+            if resp.status == 200:
+                vendor_string = yield from resp.text()
+                return vendor_string
+            # if vendor is not known to the API (404) or there
+            # was a failure during the lookup (500); set vendor
+            # to something other then None to prevent retry
+            # as the value is only relevant when it is to be stored
+            # in the 'known_devices.yaml' file which only happens
+            # the first time the device is seen.
+            return 'unknown'
+        except (asyncio.TimeoutError, aiohttp.errors.ClientError,
+                aiohttp.errors.ClientDisconnectedError):
+            # same as above
+            return 'unknown'
+        finally:
+            if resp is not None:
+                yield from resp.release()
+
 
 def load_config(path: str, hass: HomeAssistantType, consider_home: timedelta):
     """Load devices from YAML configuration file."""
@@ -483,7 +538,8 @@ def async_load_config(path: str, hass: HomeAssistantType,
         vol.Optional('gravatar', default=None): vol.Any(None, cv.string),
         vol.Optional('picture', default=None): vol.Any(None, cv.string),
         vol.Optional(CONF_CONSIDER_HOME, default=consider_home): vol.All(
-            cv.time_period, cv.positive_timedelta)
+            cv.time_period, cv.positive_timedelta),
+        vol.Optional('vendor', default=None): vol.Any(None, cv.string),
     })
     try:
         result = []
@@ -546,7 +602,8 @@ def update_config(path: str, dev_id: str, device: Device):
             'mac': device.mac,
             'picture': device.config_picture,
             'track': device.track,
-            CONF_AWAY_HIDE: device.away_hide
+            CONF_AWAY_HIDE: device.away_hide,
+            'vendor': device.vendor,
         }}
         out.write('\n')
         out.write(dump(device))

tests/components/device_tracker/test_ddwrt.py

@@ -3,6 +3,7 @@ import os
 import unittest
 from unittest import mock
 import logging
+import re
 import requests
 import requests_mock
 
@@ -17,6 +18,8 @@ from homeassistant.util import slugify
 from tests.common import (
     get_test_home_assistant, assert_setup_component, load_fixture)
 
+from ...test_util.aiohttp import mock_aiohttp_client
+
 TEST_HOST = '127.0.0.1'
 _LOGGER = logging.getLogger(__name__)
 
@@ -26,6 +29,13 @@ class TestDdwrt(unittest.TestCase):
 
     hass = None
 
+    def run(self, result=None):
+        """Mock out http calls to macvendor API for whole test suite."""
+        with mock_aiohttp_client() as aioclient_mock:
+            macvendor_re = re.compile('http://api.macvendors.com/.*')
+            aioclient_mock.get(macvendor_re, text='')
+            super().run(result)
+
     def setup_method(self, _):
         """Setup things to be run when tests are started."""
         self.hass = get_test_home_assistant()
@@ -136,6 +146,7 @@ class TestDdwrt(unittest.TestCase):
                         CONF_USERNAME: 'fake_user',
                         CONF_PASSWORD: '0'
                     }})
+                self.hass.block_till_done()
 
             path = self.hass.config.path(device_tracker.YAML_DEVICES)
             devices = config.load_yaml_config_file(path)
@@ -164,6 +175,7 @@ class TestDdwrt(unittest.TestCase):
                         CONF_USERNAME: 'fake_user',
                         CONF_PASSWORD: '0'
                     }})
+                self.hass.block_till_done()
 
             path = self.hass.config.path(device_tracker.YAML_DEVICES)
             devices = config.load_yaml_config_file(path)
@@ -192,6 +204,7 @@ class TestDdwrt(unittest.TestCase):
                         CONF_USERNAME: 'fake_user',
                         CONF_PASSWORD: '0'
                     }})
+                self.hass.block_till_done()
 
             path = self.hass.config.path(device_tracker.YAML_DEVICES)
             devices = config.load_yaml_config_file(path)

tests/components/device_tracker/test_init.py

@@ -1,5 +1,6 @@
 """The tests for the device tracker component."""
 # pylint: disable=protected-access
+import asyncio
 import json
 import logging
 import unittest
@@ -23,6 +24,8 @@ from tests.common import (
     get_test_home_assistant, fire_time_changed, fire_service_discovered,
     patch_yaml_files, assert_setup_component)
 
+from ...test_util.aiohttp import mock_aiohttp_client
+
 TEST_PLATFORM = {device_tracker.DOMAIN: {CONF_PLATFORM: 'test'}}
 
 _LOGGER = logging.getLogger(__name__)
@@ -107,6 +110,7 @@ class TestComponentsDeviceTracker(unittest.TestCase):
         self.assertEqual(device.config_picture, config.config_picture)
         self.assertEqual(device.away_hide, config.away_hide)
         self.assertEqual(device.consider_home, config.consider_home)
+        self.assertEqual(device.vendor, config.vendor)
 
     # pylint: disable=invalid-name
     @patch('homeassistant.components.device_tracker._LOGGER.warning')
@@ -154,8 +158,13 @@ class TestComponentsDeviceTracker(unittest.TestCase):
 
         self.assertTrue(setup_component(self.hass, device_tracker.DOMAIN, {
             device_tracker.DOMAIN: {CONF_PLATFORM: 'test'}}))
+
+        # wait for async calls (macvendor) to finish
+        self.hass.block_till_done()
+
         config = device_tracker.load_config(self.yaml_devices, self.hass,
                                             timedelta(seconds=0))
+
         assert len(config) == 1
         assert config[0].dev_id == 'dev1'
         assert config[0].track
@@ -181,6 +190,72 @@ class TestComponentsDeviceTracker(unittest.TestCase):
                         "55502f40dc8b7c769880b10874abc9d0.jpg?s=80&d=wavatar")
         self.assertEqual(device.config_picture, gravatar_url)
 
+    def test_mac_vendor_lookup(self):
+        """Test if vendor string is lookup on macvendors API."""
+        mac = 'B8:27:EB:00:00:00'
+        vendor_string = 'Raspberry Pi Foundation'
+
+        device = device_tracker.Device(
+            self.hass, timedelta(seconds=180), True, 'test', mac, 'Test name')
+
+        with mock_aiohttp_client() as aioclient_mock:
+            aioclient_mock.get('http://api.macvendors.com/b8:27:eb',
+                               text=vendor_string)
+
+            run_coroutine_threadsafe(device.set_vendor_for_mac(),
+                                     self.hass.loop).result()
+            assert aioclient_mock.call_count == 1
+
+        self.assertEqual(device.vendor, vendor_string)
+
+    def test_mac_vendor_lookup_unknown(self):
+        """Prevent another mac vendor lookup if was not found first time."""
+        mac = 'B8:27:EB:00:00:00'
+
+        device = device_tracker.Device(
+            self.hass, timedelta(seconds=180), True, 'test', mac, 'Test name')
+
+        with mock_aiohttp_client() as aioclient_mock:
+            aioclient_mock.get('http://api.macvendors.com/b8:27:eb',
+                               status=404)
+
+            run_coroutine_threadsafe(device.set_vendor_for_mac(),
+                                     self.hass.loop).result()
+
+            self.assertEqual(device.vendor, 'unknown')
+
+    def test_mac_vendor_lookup_error(self):
+        """Prevent another lookup if failure during API call."""
+        mac = 'B8:27:EB:00:00:00'
+
+        device = device_tracker.Device(
+            self.hass, timedelta(seconds=180), True, 'test', mac, 'Test name')
+
+        with mock_aiohttp_client() as aioclient_mock:
+            aioclient_mock.get('http://api.macvendors.com/b8:27:eb',
+                               status=500)
+
+            run_coroutine_threadsafe(device.set_vendor_for_mac(),
+                                     self.hass.loop).result()
+
+            self.assertEqual(device.vendor, 'unknown')
+
+    def test_mac_vendor_lookup_exception(self):
+        """Prevent another lookup if exception during API call."""
+        mac = 'B8:27:EB:00:00:00'
+
+        device = device_tracker.Device(
+            self.hass, timedelta(seconds=180), True, 'test', mac, 'Test name')
+
+        with mock_aiohttp_client() as aioclient_mock:
+            aioclient_mock.get('http://api.macvendors.com/b8:27:eb',
+                               exc=asyncio.TimeoutError())
+
+            run_coroutine_threadsafe(device.set_vendor_for_mac(),
+                                     self.hass.loop).result()
+
+            self.assertEqual(device.vendor, 'unknown')
+
     def test_discovery(self):
         """Test discovery."""
         scanner = get_component('device_tracker.test').SCANNER

tests/test_util/aiohttp.py

@@ -22,7 +22,8 @@ class AiohttpClientMocker:
                 text=None,
                 content=None,
                 json=None,
-                params=None):
+                params=None,
+                exc=None):
         """Mock a request."""
         if json:
             text = _json.dumps(json)
@@ -33,6 +34,8 @@ class AiohttpClientMocker:
         if params:
             url = str(yarl.URL(url).with_query(params))
 
+        self.exc = exc
+
         self._mocks.append(AiohttpClientMockResponse(
             method, url, status, content))
 
@@ -68,6 +71,9 @@ class AiohttpClientMocker:
         for response in self._mocks:
             if response.match_request(method, url, params):
                 self.mock_calls.append((method, url))
+
+                if self.exc:
+                    raise self.exc
                 return response
 
         assert False, "No mock registered for {} {}".format(method.upper(),