Update intent response (#83560)

* Add language to conversation and intent response

* Move language to intent response instead of speech

* Extend intent response for voice MVP

* Add tests for error conditions in conversation/process

* Move intent response type data into "data" field

* Move intent response error message back to speech

* Remove "success" from intent response

* Add id to target in intent response

* target defaults to None

* Update homeassistant/helpers/intent.py

* Fix test

Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

homeassistant/components/conversation/__init__.py

@@ -210,16 +210,34 @@ async def _async_converse(
 ) -> intent.IntentResponse:
     """Process text and get intent."""
     agent = await _get_agent(hass)
+    if language is None:
+        language = hass.config.language
+
     try:
         intent_result = await agent.async_process(
             text, context, conversation_id, language
         )
     except intent.IntentHandleError as err:
+        # Match was successful, but target(s) were invalid
         intent_result = intent.IntentResponse(language=language)
-        intent_result.async_set_speech(str(err))
+        intent_result.async_set_error(
+            intent.IntentResponseErrorCode.NO_VALID_TARGETS,
+            str(err),
+        )
+    except intent.IntentUnexpectedError as err:
+        # Match was successful, but an error occurred while handling intent
+        intent_result = intent.IntentResponse(language=language)
+        intent_result.async_set_error(
+            intent.IntentResponseErrorCode.FAILED_TO_HANDLE,
+            str(err),
+        )
 
     if intent_result is None:
+        # Match was not successful
         intent_result = intent.IntentResponse(language=language)
-        intent_result.async_set_speech("Sorry, I didn't understand that")
+        intent_result.async_set_error(
+            intent.IntentResponseErrorCode.NO_INTENT_MATCH,
+            "Sorry, I didn't understand that",
+        )
 
     return intent_result

homeassistant/helpers/intent.py

@@ -2,6 +2,9 @@
 from __future__ import annotations
 
 from collections.abc import Callable, Iterable
+import dataclasses
+from dataclasses import dataclass
+from enum import Enum
 import logging
 import re
 from typing import Any, TypeVar
@@ -218,9 +221,26 @@ class ServiceIntentHandler(IntentHandler):
 
         response = intent_obj.create_response()
         response.async_set_speech(self.speech.format(state.name))
+        response.async_set_target(
+            IntentResponseTarget(
+                name=state.name,
+                type=IntentResponseTargetType.ENTITY,
+                id=state.entity_id,
+            )
+        )
         return response
 
 
+class IntentCategory(Enum):
+    """Category of an intent."""
+
+    ACTION = "action"
+    """Trigger an action like turning an entity on or off"""
+
+    QUERY = "query"
+    """Get information about the state of an entity"""
+
+
 class Intent:
     """Hold the intent."""
 
@@ -232,6 +252,7 @@ class Intent:
         "text_input",
         "context",
         "language",
+        "category",
     ]
 
     def __init__(
@@ -243,6 +264,7 @@ class Intent:
         text_input: str | None,
         context: Context,
         language: str,
+        category: IntentCategory | None = None,
     ) -> None:
         """Initialize an intent."""
         self.hass = hass
@@ -252,6 +274,7 @@ class Intent:
         self.text_input = text_input
         self.context = context
         self.language = language
+        self.category = category
 
     @callback
     def create_response(self) -> IntentResponse:
@@ -259,11 +282,57 @@ class Intent:
         return IntentResponse(self, language=self.language)
 
 
+class IntentResponseType(Enum):
+    """Type of the intent response."""
+
+    ACTION_DONE = "action_done"
+    """Intent caused an action to occur"""
+
+    QUERY_ANSWER = "query_answer"
+    """Response is an answer to a query"""
+
+    ERROR = "error"
+    """Response is an error"""
+
+
+class IntentResponseErrorCode(str, Enum):
+    """Reason for an intent response error."""
+
+    NO_INTENT_MATCH = "no_intent_match"
+    """Text could not be matched to an intent"""
+
+    NO_VALID_TARGETS = "no_valid_targets"
+    """Intent was matched, but no valid areas/devices/entities were targeted"""
+
+    FAILED_TO_HANDLE = "failed_to_handle"
+    """Unexpected error occurred while handling intent"""
+
+
+class IntentResponseTargetType(str, Enum):
+    """Type of target for an intent response."""
+
+    AREA = "area"
+    DEVICE = "device"
+    ENTITY = "entity"
+    OTHER = "other"
+
+
+@dataclass
+class IntentResponseTarget:
+    """Main target of the intent response."""
+
+    name: str
+    type: IntentResponseTargetType
+    id: str | None = None
+
+
 class IntentResponse:
     """Response to an intent."""
 
     def __init__(
-        self, intent: Intent | None = None, language: str | None = None
+        self,
+        intent: Intent | None = None,
+        language: str | None = None,
     ) -> None:
         """Initialize an IntentResponse."""
         self.intent = intent
@@ -271,6 +340,14 @@ class IntentResponse:
         self.reprompt: dict[str, dict[str, Any]] = {}
         self.card: dict[str, dict[str, str]] = {}
         self.language = language
+        self.error_code: IntentResponseErrorCode | None = None
+        self.target: IntentResponseTarget | None = None
+
+        if (self.intent is not None) and (self.intent.category == IntentCategory.QUERY):
+            # speech will be the answer to the query
+            self.response_type = IntentResponseType.QUERY_ANSWER
+        else:
+            self.response_type = IntentResponseType.ACTION_DONE
 
     @callback
     def async_set_speech(
@@ -305,6 +382,20 @@ class IntentResponse:
         """Set card response."""
         self.card[card_type] = {"title": title, "content": content}
 
+    @callback
+    def async_set_error(self, code: IntentResponseErrorCode, message: str) -> None:
+        """Set response error."""
+        self.response_type = IntentResponseType.ERROR
+        self.error_code = code
+
+        # Speak error message
+        self.async_set_speech(message)
+
+    @callback
+    def async_set_target(self, target: IntentResponseTarget) -> None:
+        """Set response target."""
+        self.target = target
+
     @callback
     def as_dict(self) -> dict[str, Any]:
         """Return a dictionary representation of an intent response."""
@@ -312,9 +403,23 @@ class IntentResponse:
             "speech": self.speech,
             "card": self.card,
             "language": self.language,
+            "response_type": self.response_type.value,
         }
 
         if self.reprompt:
             response_dict["reprompt"] = self.reprompt
 
+        response_data: dict[str, Any] = {}
+
+        if self.response_type == IntentResponseType.ERROR:
+            assert self.error_code is not None, "error code is required"
+            response_data["code"] = self.error_code.value
+        else:
+            # action done or query answer
+            response_data["target"] = (
+                dataclasses.asdict(self.target) if self.target is not None else None
+            )
+
+        response_dict["data"] = response_data
+
         return response_dict

tests/components/conversation/test_init.py

@@ -1,5 +1,6 @@
 """The tests for the Conversation component."""
 from http import HTTPStatus
+from unittest.mock import patch
 
 import pytest
 
@@ -11,6 +12,14 @@ from homeassistant.setup import async_setup_component
 from tests.common import async_mock_intent, async_mock_service
 
 
+@pytest.fixture
+async def init_components(hass):
+    """Initialize relevant components with empty configs."""
+    assert await async_setup_component(hass, "homeassistant", {})
+    assert await async_setup_component(hass, "conversation", {})
+    assert await async_setup_component(hass, "intent", {})
+
+
 async def test_calling_intent(hass):
     """Test calling an intent from a conversation."""
     intents = async_mock_intent(hass, "OrderBeer")
@@ -122,6 +131,7 @@ async def test_http_processing_intent(hass, hass_client, hass_admin_user):
     data = await resp.json()
 
     assert data == {
+        "response_type": "action_done",
         "card": {
             "simple": {"content": "You chose a Grolsch.", "title": "Beer ordered"}
         },
@@ -132,18 +142,13 @@ async def test_http_processing_intent(hass, hass_client, hass_admin_user):
             }
         },
         "language": hass.config.language,
+        "data": {"target": None},
     }
 
 
 @pytest.mark.parametrize("sentence", ("turn on kitchen", "turn kitchen on"))
-async def test_turn_on_intent(hass, sentence):
+async def test_turn_on_intent(hass, init_components, sentence):
     """Test calling the turn on intent."""
-    result = await async_setup_component(hass, "homeassistant", {})
-    assert result
-
-    result = await async_setup_component(hass, "conversation", {})
-    assert result
-
     hass.states.async_set("light.kitchen", "off")
     calls = async_mock_service(hass, HASS_DOMAIN, "turn_on")
 
@@ -160,14 +165,8 @@ async def test_turn_on_intent(hass, sentence):
 
 
 @pytest.mark.parametrize("sentence", ("turn off kitchen", "turn kitchen off"))
-async def test_turn_off_intent(hass, sentence):
+async def test_turn_off_intent(hass, init_components, sentence):
     """Test calling the turn on intent."""
-    result = await async_setup_component(hass, "homeassistant", {})
-    assert result
-
-    result = await async_setup_component(hass, "conversation", {})
-    assert result
-
     hass.states.async_set("light.kitchen", "on")
     calls = async_mock_service(hass, HASS_DOMAIN, "turn_off")
 
@@ -184,14 +183,8 @@ async def test_turn_off_intent(hass, sentence):
 
 
 @pytest.mark.parametrize("sentence", ("toggle kitchen", "kitchen toggle"))
-async def test_toggle_intent(hass, sentence):
+async def test_toggle_intent(hass, init_components, sentence):
     """Test calling the turn on intent."""
-    result = await async_setup_component(hass, "homeassistant", {})
-    assert result
-
-    result = await async_setup_component(hass, "conversation", {})
-    assert result
-
     hass.states.async_set("light.kitchen", "on")
     calls = async_mock_service(hass, HASS_DOMAIN, "toggle")
 
@@ -207,12 +200,8 @@ async def test_toggle_intent(hass, sentence):
     assert call.data == {"entity_id": "light.kitchen"}
 
 
-async def test_http_api(hass, hass_client):
+async def test_http_api(hass, init_components, hass_client):
     """Test the HTTP conversation API."""
-    assert await async_setup_component(hass, "homeassistant", {})
-    assert await async_setup_component(hass, "conversation", {})
-    assert await async_setup_component(hass, "intent", {})
-
     client = await hass_client()
     hass.states.async_set("light.kitchen", "off")
     calls = async_mock_service(hass, HASS_DOMAIN, "turn_on")
@@ -221,6 +210,21 @@ async def test_http_api(hass, hass_client):
         "/api/conversation/process", json={"text": "Turn the kitchen on"}
     )
     assert resp.status == HTTPStatus.OK
+    data = await resp.json()
+
+    assert data == {
+        "card": {},
+        "speech": {"plain": {"extra_data": None, "speech": "Turned kitchen on"}},
+        "language": hass.config.language,
+        "response_type": "action_done",
+        "data": {
+            "target": {
+                "name": "kitchen",
+                "type": "entity",
+                "id": "light.kitchen",
+            }
+        },
+    }
 
     assert len(calls) == 1
     call = calls[0]
@@ -229,14 +233,96 @@ async def test_http_api(hass, hass_client):
     assert call.data == {"entity_id": "light.kitchen"}
 
 
-async def test_http_api_wrong_data(hass, hass_client):
+async def test_http_api_no_match(hass, init_components, hass_client):
+    """Test the HTTP conversation API with an intent match failure."""
+    client = await hass_client()
+
+    # Sentence should not match any intents
+    resp = await client.post("/api/conversation/process", json={"text": "do something"})
+    assert resp.status == HTTPStatus.OK
+    data = await resp.json()
+
+    assert data == {
+        "card": {},
+        "speech": {
+            "plain": {
+                "extra_data": None,
+                "speech": "Sorry, I didn't understand that",
+            },
+        },
+        "language": hass.config.language,
+        "response_type": "error",
+        "data": {
+            "code": "no_intent_match",
+        },
+    }
+
+
+async def test_http_api_no_valid_targets(hass, init_components, hass_client):
+    """Test the HTTP conversation API with no valid targets."""
+    client = await hass_client()
+
+    # No kitchen light
+    resp = await client.post(
+        "/api/conversation/process", json={"text": "turn on the kitchen"}
+    )
+    assert resp.status == HTTPStatus.OK
+    data = await resp.json()
+
+    assert data == {
+        "response_type": "error",
+        "card": {},
+        "speech": {
+            "plain": {
+                "extra_data": None,
+                "speech": "Unable to find an entity called kitchen",
+            },
+        },
+        "language": hass.config.language,
+        "data": {
+            "code": "no_valid_targets",
+        },
+    }
+
+
+async def test_http_api_handle_failure(hass, init_components, hass_client):
+    """Test the HTTP conversation API with an error during handling."""
+    client = await hass_client()
+
+    hass.states.async_set("light.kitchen", "off")
+
+    # Raise an "unexpected" error during intent handling
+    def async_handle_error(*args, **kwargs):
+        raise intent.IntentUnexpectedError(
+            "Unexpected error turning on the kitchen light"
+        )
+
+    with patch("homeassistant.helpers.intent.async_handle", new=async_handle_error):
+        resp = await client.post(
+            "/api/conversation/process", json={"text": "turn on the kitchen"}
+        )
+
+    assert resp.status == HTTPStatus.OK
+    data = await resp.json()
+
+    assert data == {
+        "response_type": "error",
+        "card": {},
+        "speech": {
+            "plain": {
+                "extra_data": None,
+                "speech": "Unexpected error turning on the kitchen light",
+            }
+        },
+        "language": hass.config.language,
+        "data": {
+            "code": "failed_to_handle",
+        },
+    }
+
+
+async def test_http_api_wrong_data(hass, init_components, hass_client):
     """Test the HTTP conversation API."""
-    result = await async_setup_component(hass, "homeassistant", {})
-    assert result
-
-    result = await async_setup_component(hass, "conversation", {})
-    assert result
-
     client = await hass_client()
 
     resp = await client.post("/api/conversation/process", json={"text": 123})
@@ -277,6 +363,7 @@ async def test_custom_agent(hass, hass_client, hass_admin_user):
     )
     assert resp.status == HTTPStatus.OK
     assert await resp.json() == {
+        "response_type": "action_done",
         "card": {},
         "speech": {
             "plain": {
@@ -285,6 +372,7 @@ async def test_custom_agent(hass, hass_client, hass_admin_user):
             }
         },
         "language": "test-language",
+        "data": {"target": None},
     }
 
     assert len(calls) == 1

tests/components/intent/test_init.py

@@ -53,6 +53,8 @@ async def test_http_handle_intent(hass, hass_client, hass_admin_user):
             }
         },
         "language": hass.config.language,
+        "response_type": "action_done",
+        "data": {"target": None},
     }