Add MQTT diagnostics (#66730)

* Add MQTT diagnostics

* Redact device tracker location

* Adjust tests

* Address comments from code review

homeassistant/components/mqtt/debug_info.py

@@ -154,7 +154,81 @@ def update_trigger_discovery_data(hass, discovery_hash, discovery_payload):
 
 def remove_trigger_discovery_data(hass, discovery_hash):
     """Remove discovery data."""
-    hass.data[DATA_MQTT_DEBUG_INFO]["triggers"][discovery_hash]["discovery_data"] = None
+    hass.data[DATA_MQTT_DEBUG_INFO]["triggers"].pop(discovery_hash)
+
+
+def _info_for_entity(hass: HomeAssistant, entity_id: str) -> dict[str, Any]:
+    mqtt_debug_info = hass.data[DATA_MQTT_DEBUG_INFO]
+    entity_info = mqtt_debug_info["entities"][entity_id]
+    subscriptions = [
+        {
+            "topic": topic,
+            "messages": [
+                {
+                    "payload": str(msg.payload),
+                    "qos": msg.qos,
+                    "retain": msg.retain,
+                    "time": msg.timestamp,
+                    "topic": msg.topic,
+                }
+                for msg in subscription["messages"]
+            ],
+        }
+        for topic, subscription in entity_info["subscriptions"].items()
+    ]
+    transmitted = [
+        {
+            "topic": topic,
+            "messages": [
+                {
+                    "payload": str(msg.payload),
+                    "qos": msg.qos,
+                    "retain": msg.retain,
+                    "time": msg.timestamp,
+                    "topic": msg.topic,
+                }
+                for msg in subscription["messages"]
+            ],
+        }
+        for topic, subscription in entity_info["transmitted"].items()
+    ]
+    discovery_data = {
+        "topic": entity_info["discovery_data"].get(ATTR_DISCOVERY_TOPIC, ""),
+        "payload": entity_info["discovery_data"].get(ATTR_DISCOVERY_PAYLOAD, ""),
+    }
+
+    return {
+        "entity_id": entity_id,
+        "subscriptions": subscriptions,
+        "discovery_data": discovery_data,
+        "transmitted": transmitted,
+    }
+
+
+def _info_for_trigger(hass: HomeAssistant, trigger_key: str) -> dict[str, Any]:
+    mqtt_debug_info = hass.data[DATA_MQTT_DEBUG_INFO]
+    trigger = mqtt_debug_info["triggers"][trigger_key]
+    discovery_data = None
+    if trigger["discovery_data"] is not None:
+        discovery_data = {
+            "topic": trigger["discovery_data"][ATTR_DISCOVERY_TOPIC],
+            "payload": trigger["discovery_data"][ATTR_DISCOVERY_PAYLOAD],
+        }
+    return {"discovery_data": discovery_data, "trigger_key": trigger_key}
+
+
+def info_for_config_entry(hass):
+    """Get debug info for all entities and triggers."""
+    mqtt_info = {"entities": [], "triggers": []}
+    mqtt_debug_info = hass.data[DATA_MQTT_DEBUG_INFO]
+
+    for entity_id in mqtt_debug_info["entities"]:
+        mqtt_info["entities"].append(_info_for_entity(hass, entity_id))
+
+    for trigger_key in mqtt_debug_info["triggers"]:
+        mqtt_info["triggers"].append(_info_for_trigger(hass, trigger_key))
+
+    return mqtt_info
 
 
 def info_for_device(hass, device_id):
@@ -170,60 +244,12 @@ def info_for_device(hass, device_id):
         if entry.entity_id not in mqtt_debug_info["entities"]:
             continue
 
-        entity_info = mqtt_debug_info["entities"][entry.entity_id]
+        mqtt_info["entities"].append(_info_for_entity(hass, entry.entity_id))
-        subscriptions = [
-            {
-                "topic": topic,
-                "messages": [
-                    {
-                        "payload": str(msg.payload),
-                        "qos": msg.qos,
-                        "retain": msg.retain,
-                        "time": msg.timestamp,
-                        "topic": msg.topic,
-                    }
-                    for msg in list(subscription["messages"])
-                ],
-            }
-            for topic, subscription in entity_info["subscriptions"].items()
-        ]
-        transmitted = [
-            {
-                "topic": topic,
-                "messages": [
-                    {
-                        "payload": str(msg.payload),
-                        "qos": msg.qos,
-                        "retain": msg.retain,
-                        "time": msg.timestamp,
-                        "topic": msg.topic,
-                    }
-                    for msg in list(subscription["messages"])
-                ],
-            }
-            for topic, subscription in entity_info["transmitted"].items()
-        ]
-        discovery_data = {
-            "topic": entity_info["discovery_data"].get(ATTR_DISCOVERY_TOPIC, ""),
-            "payload": entity_info["discovery_data"].get(ATTR_DISCOVERY_PAYLOAD, ""),
-        }
-        mqtt_info["entities"].append(
-            {
-                "entity_id": entry.entity_id,
-                "subscriptions": subscriptions,
-                "discovery_data": discovery_data,
-                "transmitted": transmitted,
-            }
-        )
 
-    for trigger in mqtt_debug_info["triggers"].values():
+    for trigger_key, trigger in mqtt_debug_info["triggers"].items():
-        if trigger["device_id"] != device_id or trigger["discovery_data"] is None:
+        if trigger["device_id"] != device_id:
             continue
 
-        discovery_data = {
+        mqtt_info["triggers"].append(_info_for_trigger(hass, trigger_key))
-            "topic": trigger["discovery_data"][ATTR_DISCOVERY_TOPIC],
-            "payload": trigger["discovery_data"][ATTR_DISCOVERY_PAYLOAD],
-        }
-        mqtt_info["triggers"].append({"discovery_data": discovery_data})
 
     return mqtt_info

homeassistant/components/mqtt/diagnostics.py

@@ -0,0 +1,126 @@
+"""Diagnostics support for MQTT."""
+from __future__ import annotations
+
+from typing import Any
+
+from homeassistant.components import device_tracker
+from homeassistant.components.diagnostics import async_redact_data
+from homeassistant.config_entries import ConfigEntry
+from homeassistant.const import (
+    ATTR_LATITUDE,
+    ATTR_LONGITUDE,
+    CONF_PASSWORD,
+    CONF_USERNAME,
+)
+from homeassistant.core import HomeAssistant, callback, split_entity_id
+from homeassistant.helpers import device_registry as dr, entity_registry as er
+from homeassistant.helpers.device_registry import DeviceEntry
+
+from . import DATA_MQTT, MQTT, debug_info, is_connected
+
+REDACT_CONFIG = {CONF_PASSWORD, CONF_USERNAME}
+REDACT_STATE_DEVICE_TRACKER = {ATTR_LATITUDE, ATTR_LONGITUDE}
+
+
+async def async_get_config_entry_diagnostics(
+    hass: HomeAssistant, entry: ConfigEntry
+) -> dict[str, Any]:
+    """Return diagnostics for a config entry."""
+    return _async_get_diagnostics(hass, entry)
+
+
+async def async_get_device_diagnostics(
+    hass: HomeAssistant, entry: ConfigEntry, device: DeviceEntry
+) -> dict[str, Any]:
+    """Return diagnostics for a device entry."""
+    return _async_get_diagnostics(hass, entry, device)
+
+
+@callback
+def _async_get_diagnostics(
+    hass: HomeAssistant,
+    entry: ConfigEntry,
+    device: DeviceEntry | None = None,
+) -> dict[str, Any]:
+    """Return diagnostics for a config entry."""
+    mqtt_instance: MQTT = hass.data[DATA_MQTT]
+
+    redacted_config = async_redact_data(mqtt_instance.conf, REDACT_CONFIG)
+
+    data = {
+        "connected": is_connected(hass),
+        "mqtt_config": redacted_config,
+    }
+
+    if device:
+        data["device"] = _async_device_as_dict(hass, device)
+        data["mqtt_debug_info"] = debug_info.info_for_device(hass, device.id)
+    else:
+        device_registry = dr.async_get(hass)
+        data.update(
+            devices=[
+                _async_device_as_dict(hass, device)
+                for device in dr.async_entries_for_config_entry(
+                    device_registry, entry.entry_id
+                )
+            ],
+            mqtt_debug_info=debug_info.info_for_config_entry(hass),
+        )
+
+    return data
+
+
+@callback
+def _async_device_as_dict(hass: HomeAssistant, device: DeviceEntry) -> dict[str, Any]:
+    """Represent an MQTT device as a dictionary."""
+
+    # Gather information how this MQTT device is represented in Home Assistant
+    entity_registry = er.async_get(hass)
+    data: dict[str, Any] = {
+        "id": device.id,
+        "name": device.name,
+        "name_by_user": device.name_by_user,
+        "disabled": device.disabled,
+        "disabled_by": device.disabled_by,
+        "entities": [],
+    }
+
+    entities = er.async_entries_for_device(
+        entity_registry,
+        device_id=device.id,
+        include_disabled_entities=True,
+    )
+
+    for entity_entry in entities:
+        state = hass.states.get(entity_entry.entity_id)
+        state_dict = None
+        if state:
+            state_dict = dict(state.as_dict())
+
+            # The context doesn't provide useful information in this case.
+            state_dict.pop("context", None)
+
+            entity_domain = split_entity_id(state.entity_id)[0]
+
+            # Retract some sensitive state attributes
+            if entity_domain == device_tracker.DOMAIN:
+                state_dict["attributes"] = async_redact_data(
+                    state_dict["attributes"], REDACT_STATE_DEVICE_TRACKER
+                )
+
+        data["entities"].append(
+            {
+                "device_class": entity_entry.device_class,
+                "disabled_by": entity_entry.disabled_by,
+                "disabled": entity_entry.disabled,
+                "entity_category": entity_entry.entity_category,
+                "entity_id": entity_entry.entity_id,
+                "icon": entity_entry.icon,
+                "original_device_class": entity_entry.original_device_class,
+                "original_icon": entity_entry.original_icon,
+                "state": state_dict,
+                "unit_of_measurement": entity_entry.unit_of_measurement,
+            }
+        )
+
+    return data

tests/components/mqtt/test_diagnostics.py

@@ -0,0 +1,263 @@
+"""Test MQTT diagnostics."""
+
+import json
+from unittest.mock import ANY
+
+import pytest
+
+from homeassistant.components import mqtt
+
+from tests.common import async_fire_mqtt_message, mock_device_registry
+from tests.components.diagnostics import (
+    get_diagnostics_for_config_entry,
+    get_diagnostics_for_device,
+)
+
+default_config = {
+    "birth_message": {},
+    "broker": "mock-broker",
+    "discovery": True,
+    "discovery_prefix": "homeassistant",
+    "keepalive": 60,
+    "port": 1883,
+    "protocol": "3.1.1",
+    "tls_version": "auto",
+    "will_message": {
+        "payload": "offline",
+        "qos": 0,
+        "retain": False,
+        "topic": "homeassistant/status",
+    },
+}
+
+
+@pytest.fixture
+def device_reg(hass):
+    """Return an empty, loaded, registry."""
+    return mock_device_registry(hass)
+
+
+async def test_entry_diagnostics(hass, device_reg, hass_client, mqtt_mock):
+    """Test config entry diagnostics."""
+    config_entry = hass.config_entries.async_entries(mqtt.DOMAIN)[0]
+    mqtt_mock.connected = True
+
+    assert await get_diagnostics_for_config_entry(hass, hass_client, config_entry) == {
+        "connected": True,
+        "devices": [],
+        "mqtt_config": default_config,
+        "mqtt_debug_info": {"entities": [], "triggers": []},
+    }
+
+    # Discover a device with an entity and a trigger
+    config_sensor = {
+        "device": {"identifiers": ["0AFFD2"]},
+        "platform": "mqtt",
+        "state_topic": "foobar/sensor",
+        "unique_id": "unique",
+    }
+    config_trigger = {
+        "automation_type": "trigger",
+        "device": {"identifiers": ["0AFFD2"]},
+        "platform": "mqtt",
+        "topic": "test-topic1",
+        "type": "foo",
+        "subtype": "bar",
+    }
+    data_sensor = json.dumps(config_sensor)
+    data_trigger = json.dumps(config_trigger)
+
+    async_fire_mqtt_message(hass, "homeassistant/sensor/bla/config", data_sensor)
+    async_fire_mqtt_message(
+        hass, "homeassistant/device_automation/bla/config", data_trigger
+    )
+    await hass.async_block_till_done()
+
+    device_entry = device_reg.async_get_device({("mqtt", "0AFFD2")})
+
+    expected_debug_info = {
+        "entities": [
+            {
+                "entity_id": "sensor.mqtt_sensor",
+                "subscriptions": [{"topic": "foobar/sensor", "messages": []}],
+                "discovery_data": {
+                    "payload": config_sensor,
+                    "topic": "homeassistant/sensor/bla/config",
+                },
+                "transmitted": [],
+            }
+        ],
+        "triggers": [
+            {
+                "discovery_data": {
+                    "payload": config_trigger,
+                    "topic": "homeassistant/device_automation/bla/config",
+                },
+                "trigger_key": ["device_automation", "bla"],
+            }
+        ],
+    }
+
+    expected_device = {
+        "disabled": False,
+        "disabled_by": None,
+        "entities": [
+            {
+                "device_class": None,
+                "disabled": False,
+                "disabled_by": None,
+                "entity_category": None,
+                "entity_id": "sensor.mqtt_sensor",
+                "icon": None,
+                "original_device_class": None,
+                "original_icon": None,
+                "state": {
+                    "attributes": {"friendly_name": "MQTT Sensor"},
+                    "entity_id": "sensor.mqtt_sensor",
+                    "last_changed": ANY,
+                    "last_updated": ANY,
+                    "state": "unknown",
+                },
+                "unit_of_measurement": None,
+            }
+        ],
+        "id": device_entry.id,
+        "name": None,
+        "name_by_user": None,
+    }
+
+    assert await get_diagnostics_for_config_entry(hass, hass_client, config_entry) == {
+        "connected": True,
+        "devices": [expected_device],
+        "mqtt_config": default_config,
+        "mqtt_debug_info": expected_debug_info,
+    }
+
+    assert await get_diagnostics_for_device(
+        hass, hass_client, config_entry, device_entry
+    ) == {
+        "connected": True,
+        "device": expected_device,
+        "mqtt_config": default_config,
+        "mqtt_debug_info": expected_debug_info,
+    }
+
+
+@pytest.mark.parametrize(
+    "mqtt_config",
+    [
+        {
+            mqtt.CONF_BROKER: "mock-broker",
+            mqtt.CONF_BIRTH_MESSAGE: {},
+            mqtt.CONF_PASSWORD: "hunter2",
+            mqtt.CONF_USERNAME: "my_user",
+        }
+    ],
+)
+async def test_redact_diagnostics(hass, device_reg, hass_client, mqtt_mock):
+    """Test redacting diagnostics."""
+    expected_config = dict(default_config)
+    expected_config["password"] = "**REDACTED**"
+    expected_config["username"] = "**REDACTED**"
+
+    config_entry = hass.config_entries.async_entries(mqtt.DOMAIN)[0]
+    mqtt_mock.connected = True
+
+    # Discover a device with a device tracker
+    config_tracker = {
+        "device": {"identifiers": ["0AFFD2"]},
+        "platform": "mqtt",
+        "state_topic": "foobar/device_tracker",
+        "json_attributes_topic": "attributes-topic",
+        "unique_id": "unique",
+    }
+    data_tracker = json.dumps(config_tracker)
+
+    async_fire_mqtt_message(
+        hass, "homeassistant/device_tracker/bla/config", data_tracker
+    )
+    await hass.async_block_till_done()
+
+    location_data = '{"latitude":32.87336,"longitude": -117.22743, "gps_accuracy":1.5}'
+    async_fire_mqtt_message(hass, "attributes-topic", location_data)
+    await hass.async_block_till_done()
+
+    device_entry = device_reg.async_get_device({("mqtt", "0AFFD2")})
+
+    expected_debug_info = {
+        "entities": [
+            {
+                "entity_id": "device_tracker.mqtt_unique",
+                "subscriptions": [
+                    {
+                        "topic": "attributes-topic",
+                        "messages": [
+                            {
+                                "payload": location_data,
+                                "qos": 0,
+                                "retain": False,
+                                "time": ANY,
+                                "topic": "attributes-topic",
+                            }
+                        ],
+                    },
+                    {"topic": "foobar/device_tracker", "messages": []},
+                ],
+                "discovery_data": {
+                    "payload": config_tracker,
+                    "topic": "homeassistant/device_tracker/bla/config",
+                },
+                "transmitted": [],
+            }
+        ],
+        "triggers": [],
+    }
+
+    expected_device = {
+        "disabled": False,
+        "disabled_by": None,
+        "entities": [
+            {
+                "device_class": None,
+                "disabled": False,
+                "disabled_by": None,
+                "entity_category": None,
+                "entity_id": "device_tracker.mqtt_unique",
+                "icon": None,
+                "original_device_class": None,
+                "original_icon": None,
+                "state": {
+                    "attributes": {
+                        "gps_accuracy": 1.5,
+                        "latitude": "**REDACTED**",
+                        "longitude": "**REDACTED**",
+                        "source_type": None,
+                    },
+                    "entity_id": "device_tracker.mqtt_unique",
+                    "last_changed": ANY,
+                    "last_updated": ANY,
+                    "state": "home",
+                },
+                "unit_of_measurement": None,
+            }
+        ],
+        "id": device_entry.id,
+        "name": None,
+        "name_by_user": None,
+    }
+
+    assert await get_diagnostics_for_config_entry(hass, hass_client, config_entry) == {
+        "connected": True,
+        "devices": [expected_device],
+        "mqtt_config": expected_config,
+        "mqtt_debug_info": expected_debug_info,
+    }
+
+    assert await get_diagnostics_for_device(
+        hass, hass_client, config_entry, device_entry
+    ) == {
+        "connected": True,
+        "device": expected_device,
+        "mqtt_config": expected_config,
+        "mqtt_debug_info": expected_debug_info,
+    }

tests/components/mqtt/test_init.py

@@ -1529,15 +1529,27 @@ async def test_mqtt_ws_get_device_debug_info(
     hass, device_reg, hass_ws_client, mqtt_mock
 ):
     """Test MQTT websocket device debug info."""
-    config = {
+    config_sensor = {
         "device": {"identifiers": ["0AFFD2"]},
         "platform": "mqtt",
         "state_topic": "foobar/sensor",
         "unique_id": "unique",
     }
-    data = json.dumps(config)
+    config_trigger = {
+        "automation_type": "trigger",
+        "device": {"identifiers": ["0AFFD2"]},
+        "platform": "mqtt",
+        "topic": "test-topic1",
+        "type": "foo",
+        "subtype": "bar",
+    }
+    data_sensor = json.dumps(config_sensor)
+    data_trigger = json.dumps(config_trigger)
 
-    async_fire_mqtt_message(hass, "homeassistant/sensor/bla/config", data)
+    async_fire_mqtt_message(hass, "homeassistant/sensor/bla/config", data_sensor)
+    async_fire_mqtt_message(
+        hass, "homeassistant/device_automation/bla/config", data_trigger
+    )
     await hass.async_block_till_done()
 
     # Verify device entry is created
@@ -1556,13 +1568,21 @@ async def test_mqtt_ws_get_device_debug_info(
                 "entity_id": "sensor.mqtt_sensor",
                 "subscriptions": [{"topic": "foobar/sensor", "messages": []}],
                 "discovery_data": {
-                    "payload": config,
+                    "payload": config_sensor,
                     "topic": "homeassistant/sensor/bla/config",
                 },
                 "transmitted": [],
             }
         ],
-        "triggers": [],
+        "triggers": [
+            {
+                "discovery_data": {
+                    "payload": config_trigger,
+                    "topic": "homeassistant/device_automation/bla/config",
+                },
+                "trigger_key": ["device_automation", "bla"],
+            }
+        ],
     }
     assert response["result"] == expected_result
 

tests/conftest.py

@@ -609,6 +609,7 @@ async def mqtt_mock(hass, mqtt_client_mock, mqtt_config):
         spec_set=hass.data["mqtt"],
         wraps=hass.data["mqtt"],
     )
+    mqtt_component_mock.conf = hass.data["mqtt"].conf  # For diagnostics
     mqtt_component_mock._mqttc = mqtt_client_mock
 
     hass.data["mqtt"] = mqtt_component_mock