Add LLM tools (#115464)

* Add llm helper

* break out Tool.specification as class members

* Format state output

* Fix intent tests

* Removed auto initialization of intents - let conversation platforms do that

* Handle DynamicServiceIntentHandler.extra_slots

* Add optional description to IntentTool init

* Add device_id and conversation_id parameters

* intent tests

* Add LLM tools tests

* coverage

* add agent_id parameter

* Apply suggestions from code review

* Apply suggestions from code review

* Apply suggestions from code review

* Apply suggestions from code review

* Apply suggestions from code review

* Fix tests

* Fix intent schema

* Allow a Python function to be registered as am LLM tool

* Add IntentHandler.effective_slot_schema

* Ensure IntentHandler.slot_schema to be vol.Schema

* Raise meaningful error on tool not found

* Move this change to a separate PR

* Update todo integration intent

* Remove Tool constructor

* Move IntentTool to intent helper

* Convert custom serializer into class method

* Remove tool_input from FunctionTool auto arguments to avoid recursion

* Remove conversion into Open API format

* Apply suggestions from code review

* Fix tests

* Use HassKey for helpers (see #117012)

* Add support for functions with typed lists, dicts, and sets as type hints

* Remove FunctionTool

* Added API to get registered intents

* Move IntentTool to the llm library

* Return only handlers in intents.async.get

* Removed llm tool registration from intent library

* Removed tool registration

* Add bind_hass back for now

* removed area and floor resolving

* fix test

* Apply suggestions from code review

* Improve coverage

* Fix intent_type type

* Temporary disable HassClimateGetTemperature intent

* Remove bind_hass

* Fix usage of slot schema

* Fix test

* Revert some test changes

* Don't mutate tool_input

---------

Co-authored-by: Paulus Schoutsen <paulus@home-assistant.io>
Co-authored-by: Paulus Schoutsen <balloob@gmail.com>

homeassistant/helpers/intent.py

@@ -87,6 +87,12 @@ def async_remove(hass: HomeAssistant, intent_type: str) -> None:
     intents.pop(intent_type, None)
 
 
+@callback
+def async_get(hass: HomeAssistant) -> Iterable[IntentHandler]:
+    """Return registered intents."""
+    return hass.data.get(DATA_KEY, {}).values()
+
+
 @bind_hass
 async def async_handle(
     hass: HomeAssistant,

homeassistant/helpers/llm.py

@@ -0,0 +1,122 @@
+"""Module to coordinate llm tools."""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from collections.abc import Iterable
+from dataclasses import dataclass
+import logging
+from typing import Any
+
+import voluptuous as vol
+
+from homeassistant.components.climate.intent import INTENT_GET_TEMPERATURE
+from homeassistant.components.weather.intent import INTENT_GET_WEATHER
+from homeassistant.core import Context, HomeAssistant, callback
+from homeassistant.exceptions import HomeAssistantError
+from homeassistant.util.json import JsonObjectType
+
+from . import intent
+
+_LOGGER = logging.getLogger(__name__)
+
+IGNORE_INTENTS = [
+    intent.INTENT_NEVERMIND,
+    intent.INTENT_GET_STATE,
+    INTENT_GET_WEATHER,
+    INTENT_GET_TEMPERATURE,
+]
+
+
+@dataclass(slots=True)
+class ToolInput:
+    """Tool input to be processed."""
+
+    tool_name: str
+    tool_args: dict[str, Any]
+    platform: str
+    context: Context | None
+    user_prompt: str | None
+    language: str | None
+    assistant: str | None
+
+
+class Tool:
+    """LLM Tool base class."""
+
+    name: str
+    description: str | None = None
+    parameters: vol.Schema = vol.Schema({})
+
+    @abstractmethod
+    async def async_call(
+        self, hass: HomeAssistant, tool_input: ToolInput
+    ) -> JsonObjectType:
+        """Call the tool."""
+        raise NotImplementedError
+
+    def __repr__(self) -> str:
+        """Represent a string of a Tool."""
+        return f"<{self.__class__.__name__} - {self.name}>"
+
+
+@callback
+def async_get_tools(hass: HomeAssistant) -> Iterable[Tool]:
+    """Return a list of LLM tools."""
+    for intent_handler in intent.async_get(hass):
+        if intent_handler.intent_type not in IGNORE_INTENTS:
+            yield IntentTool(intent_handler)
+
+
+@callback
+async def async_call_tool(hass: HomeAssistant, tool_input: ToolInput) -> JsonObjectType:
+    """Call a LLM tool, validate args and return the response."""
+    for tool in async_get_tools(hass):
+        if tool.name == tool_input.tool_name:
+            break
+    else:
+        raise HomeAssistantError(f'Tool "{tool_input.tool_name}" not found')
+
+    _tool_input = ToolInput(
+        tool_name=tool.name,
+        tool_args=tool.parameters(tool_input.tool_args),
+        platform=tool_input.platform,
+        context=tool_input.context or Context(),
+        user_prompt=tool_input.user_prompt,
+        language=tool_input.language,
+        assistant=tool_input.assistant,
+    )
+
+    return await tool.async_call(hass, _tool_input)
+
+
+class IntentTool(Tool):
+    """LLM Tool representing an Intent."""
+
+    def __init__(
+        self,
+        intent_handler: intent.IntentHandler,
+    ) -> None:
+        """Init the class."""
+        self.name = intent_handler.intent_type
+        self.description = f"Execute Home Assistant {self.name} intent"
+        if slot_schema := intent_handler.slot_schema:
+            self.parameters = vol.Schema(slot_schema)
+
+    async def async_call(
+        self, hass: HomeAssistant, tool_input: ToolInput
+    ) -> JsonObjectType:
+        """Handle the intent."""
+        slots = {key: {"value": val} for key, val in tool_input.tool_args.items()}
+
+        intent_response = await intent.async_handle(
+            hass,
+            tool_input.platform,
+            self.name,
+            slots,
+            tool_input.user_prompt,
+            tool_input.context,
+            tool_input.language,
+            tool_input.assistant,
+        )
+        return intent_response.as_dict()

tests/helpers/test_intent.py

@@ -610,7 +610,7 @@ def test_async_register(hass: HomeAssistant) -> None:
 
     intent.async_register(hass, handler)
 
-    assert hass.data[intent.DATA_KEY]["test_intent"] == handler
+    assert list(intent.async_get(hass)) == [handler]
 
 
 def test_async_register_overwrite(hass: HomeAssistant) -> None:
@@ -629,7 +629,7 @@ def test_async_register_overwrite(hass: HomeAssistant) -> None:
             "Intent %s is being overwritten by %s", "test_intent", handler2
         )
 
-    assert hass.data[intent.DATA_KEY]["test_intent"] == handler2
+    assert list(intent.async_get(hass)) == [handler2]
 
 
 def test_async_remove(hass: HomeAssistant) -> None:
@@ -640,7 +640,7 @@ def test_async_remove(hass: HomeAssistant) -> None:
     intent.async_register(hass, handler)
     intent.async_remove(hass, "test_intent")
 
-    assert "test_intent" not in hass.data[intent.DATA_KEY]
+    assert not list(intent.async_get(hass))
 
 
 def test_async_remove_no_existing_entry(hass: HomeAssistant) -> None:
@@ -651,7 +651,7 @@ def test_async_remove_no_existing_entry(hass: HomeAssistant) -> None:
 
     intent.async_remove(hass, "test_intent2")
 
-    assert "test_intent2" not in hass.data[intent.DATA_KEY]
+    assert list(intent.async_get(hass)) == [handler]
 
 
 def test_async_remove_no_existing(hass: HomeAssistant) -> None:

tests/helpers/test_llm.py

@@ -0,0 +1,94 @@
+"""Tests for the llm helpers."""
+
+from unittest.mock import patch
+
+import pytest
+import voluptuous as vol
+
+from homeassistant.core import Context, HomeAssistant, State
+from homeassistant.exceptions import HomeAssistantError
+from homeassistant.helpers import config_validation as cv, intent, llm
+
+
+async def test_call_tool_no_existing(hass: HomeAssistant) -> None:
+    """Test calling an llm tool where no config exists."""
+    with pytest.raises(HomeAssistantError):
+        await llm.async_call_tool(
+            hass,
+            llm.ToolInput(
+                "test_tool",
+                {},
+                "test_platform",
+                None,
+                None,
+                None,
+                None,
+            ),
+        )
+
+
+async def test_intent_tool(hass: HomeAssistant) -> None:
+    """Test IntentTool class."""
+    schema = {
+        vol.Optional("area"): cv.string,
+        vol.Optional("floor"): cv.string,
+    }
+
+    class MyIntentHandler(intent.IntentHandler):
+        intent_type = "test_intent"
+        slot_schema = schema
+
+    intent_handler = MyIntentHandler()
+
+    intent.async_register(hass, intent_handler)
+
+    assert len(list(llm.async_get_tools(hass))) == 1
+    tool = list(llm.async_get_tools(hass))[0]
+    assert tool.name == "test_intent"
+    assert tool.description == "Execute Home Assistant test_intent intent"
+    assert tool.parameters == vol.Schema(intent_handler.slot_schema)
+    assert str(tool) == "<IntentTool - test_intent>"
+
+    test_context = Context()
+    intent_response = intent.IntentResponse("*")
+    intent_response.matched_states = [State("light.matched", "on")]
+    intent_response.unmatched_states = [State("light.unmatched", "on")]
+    tool_input = llm.ToolInput(
+        tool_name="test_intent",
+        tool_args={"area": "kitchen", "floor": "ground_floor"},
+        platform="test_platform",
+        context=test_context,
+        user_prompt="test_text",
+        language="*",
+        assistant="test_assistant",
+    )
+
+    with patch(
+        "homeassistant.helpers.intent.async_handle", return_value=intent_response
+    ) as mock_intent_handle:
+        response = await llm.async_call_tool(hass, tool_input)
+
+    mock_intent_handle.assert_awaited_once_with(
+        hass,
+        "test_platform",
+        "test_intent",
+        {
+            "area": {"value": "kitchen"},
+            "floor": {"value": "ground_floor"},
+        },
+        "test_text",
+        test_context,
+        "*",
+        "test_assistant",
+    )
+    assert response == {
+        "card": {},
+        "data": {
+            "failed": [],
+            "success": [],
+            "targets": [],
+        },
+        "language": "*",
+        "response_type": "action_done",
+        "speech": {},
+    }