Store automation and script traces (#56894)

* Store automation and script traces

* Pylint

* Deduplicate code

* Fix issues when no stored traces are available

* Store serialized data for restored traces

* Update WS API

* Update test

* Restore context

* Improve tests

* Add new test files

* Rename restore_traces to async_restore_traces

* Refactor trace.websocket_api

* Defer loading stored traces

* Lint

* Revert refactoring which is no longer needed

* Correct order when restoring traces

* Apply suggestion from code review

* Improve test coverage

* Apply suggestions from code review

homeassistant/components/automation/__init__.py

@@ -228,7 +228,6 @@ def areas_in_automation(hass: HomeAssistant, entity_id: str) -> list[str]:
 
 async def async_setup(hass, config):
     """Set up all automations."""
-    # Local import to avoid circular import
     hass.data[DOMAIN] = component = EntityComponent(LOGGER, DOMAIN, hass)
 
     # To register the automation blueprints

homeassistant/components/automation/trace.py

@@ -8,6 +8,8 @@ from homeassistant.components.trace import ActionTrace, async_store_trace
 from homeassistant.components.trace.const import CONF_STORED_TRACES
 from homeassistant.core import Context
 
+from .const import DOMAIN
+
 # mypy: allow-untyped-calls, allow-untyped-defs
 # mypy: no-check-untyped-defs, no-warn-return-any
 
@@ -15,6 +17,8 @@ from homeassistant.core import Context
 class AutomationTrace(ActionTrace):
     """Container for automation trace."""
 
+    _domain = DOMAIN
+
     def __init__(
         self,
         item_id: str,
@@ -23,8 +27,7 @@ class AutomationTrace(ActionTrace):
         context: Context,
     ) -> None:
         """Container for automation trace."""
-        key = ("automation", item_id)
+        super().__init__(item_id, config, blueprint_inputs, context)
-        super().__init__(key, config, blueprint_inputs, context)
         self._trigger_description: str | None = None
 
     def set_trigger_description(self, trigger: str) -> None:
@@ -33,6 +36,9 @@ class AutomationTrace(ActionTrace):
 
     def as_short_dict(self) -> dict[str, Any]:
         """Return a brief dictionary version of this AutomationTrace."""
+        if self._short_dict:
+            return self._short_dict
+
         result = super().as_short_dict()
         result["trigger"] = self._trigger_description
         return result

homeassistant/components/script/trace.py

@@ -9,20 +9,13 @@ from homeassistant.components.trace import ActionTrace, async_store_trace
 from homeassistant.components.trace.const import CONF_STORED_TRACES
 from homeassistant.core import Context, HomeAssistant
 
+from .const import DOMAIN
+
 
 class ScriptTrace(ActionTrace):
-    """Container for automation trace."""
+    """Container for script trace."""
 
-    def __init__(
+    _domain = DOMAIN
-        self,
-        item_id: str,
-        config: dict[str, Any],
-        blueprint_inputs: dict[str, Any],
-        context: Context,
-    ) -> None:
-        """Container for automation trace."""
-        key = ("script", item_id)
-        super().__init__(key, config, blueprint_inputs, context)
 
 
 @contextmanager

homeassistant/components/trace/__init__.py

@@ -1,15 +1,20 @@
 """Support for script and automation tracing and debugging."""
 from __future__ import annotations
 
+import abc
 from collections import deque
 import datetime as dt
-from itertools import count
+import logging
 from typing import Any
 
 import voluptuous as vol
 
+from homeassistant.const import EVENT_HOMEASSISTANT_STOP
 from homeassistant.core import Context
+from homeassistant.exceptions import HomeAssistantError
 import homeassistant.helpers.config_validation as cv
+from homeassistant.helpers.json import ExtendedJSONEncoder
+from homeassistant.helpers.storage import Store
 from homeassistant.helpers.trace import (
     TraceElement,
     script_execution_get,
@@ -18,13 +23,25 @@ from homeassistant.helpers.trace import (
     trace_set_child_id,
 )
 import homeassistant.util.dt as dt_util
+import homeassistant.util.uuid as uuid_util
 
 from . import websocket_api
-from .const import CONF_STORED_TRACES, DATA_TRACE, DEFAULT_STORED_TRACES
+from .const import (
+    CONF_STORED_TRACES,
+    DATA_TRACE,
+    DATA_TRACE_STORE,
+    DATA_TRACES_RESTORED,
+    DEFAULT_STORED_TRACES,
+)
 from .utils import LimitedSizeDict
 
+_LOGGER = logging.getLogger(__name__)
+
 DOMAIN = "trace"
 
+STORAGE_KEY = "trace.saved_traces"
+STORAGE_VERSION = 1
+
 TRACE_CONFIG_SCHEMA = {
     vol.Optional(CONF_STORED_TRACES, default=DEFAULT_STORED_TRACES): cv.positive_int
 }
@@ -34,13 +51,89 @@ async def async_setup(hass, config):
     """Initialize the trace integration."""
     hass.data[DATA_TRACE] = {}
     websocket_api.async_setup(hass)
+    store = Store(hass, STORAGE_VERSION, STORAGE_KEY, encoder=ExtendedJSONEncoder)
+    hass.data[DATA_TRACE_STORE] = store
+
+    async def _async_store_traces_at_stop(*_) -> None:
+        """Save traces to storage."""
+        _LOGGER.debug("Storing traces")
+        try:
+            await store.async_save(
+                {
+                    key: list(traces.values())
+                    for key, traces in hass.data[DATA_TRACE].items()
+                }
+            )
+        except HomeAssistantError as exc:
+            _LOGGER.error("Error storing traces", exc_info=exc)
+
+    # Store traces when stopping hass
+    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, _async_store_traces_at_stop)
+
     return True
 
 
+async def async_get_trace(hass, key, run_id):
+    """Return the requested trace."""
+    # Restore saved traces if not done
+    await async_restore_traces(hass)
+
+    return hass.data[DATA_TRACE][key][run_id].as_extended_dict()
+
+
+async def async_list_contexts(hass, key):
+    """List contexts for which we have traces."""
+    # Restore saved traces if not done
+    await async_restore_traces(hass)
+
+    if key is not None:
+        values = {key: hass.data[DATA_TRACE].get(key, {})}
+    else:
+        values = hass.data[DATA_TRACE]
+
+    def _trace_id(run_id, key) -> dict:
+        """Make trace_id for the response."""
+        domain, item_id = key.split(".", 1)
+        return {"run_id": run_id, "domain": domain, "item_id": item_id}
+
+    return {
+        trace.context.id: _trace_id(trace.run_id, key)
+        for key, traces in values.items()
+        for trace in traces.values()
+    }
+
+
+def _get_debug_traces(hass, key):
+    """Return a serializable list of debug traces for a script or automation."""
+    traces = []
+
+    for trace in hass.data[DATA_TRACE].get(key, {}).values():
+        traces.append(trace.as_short_dict())
+
+    return traces
+
+
+async def async_list_traces(hass, wanted_domain, wanted_key):
+    """List traces for a domain."""
+    # Restore saved traces if not done already
+    await async_restore_traces(hass)
+
+    if not wanted_key:
+        traces = []
+        for key in hass.data[DATA_TRACE]:
+            domain = key.split(".", 1)[0]
+            if domain == wanted_domain:
+                traces.extend(_get_debug_traces(hass, key))
+    else:
+        traces = _get_debug_traces(hass, wanted_key)
+
+    return traces
+
+
 def async_store_trace(hass, trace, stored_traces):
-    """Store a trace if its item_id is valid."""
+    """Store a trace if its key is valid."""
     key = trace.key
-    if key[1]:
+    if key:
         traces = hass.data[DATA_TRACE]
         if key not in traces:
             traces[key] = LimitedSizeDict(size_limit=stored_traces)
@@ -49,14 +142,79 @@ def async_store_trace(hass, trace, stored_traces):
         traces[key][trace.run_id] = trace
 
 
-class ActionTrace:
+def _async_store_restored_trace(hass, trace):
+    """Store a restored trace and move it to the end of the LimitedSizeDict."""
+    key = trace.key
+    traces = hass.data[DATA_TRACE]
+    if key not in traces:
+        traces[key] = LimitedSizeDict()
+    traces[key][trace.run_id] = trace
+    traces[key].move_to_end(trace.run_id, last=False)
+
+
+async def async_restore_traces(hass):
+    """Restore saved traces."""
+    if DATA_TRACES_RESTORED in hass.data:
+        return
+
+    hass.data[DATA_TRACES_RESTORED] = True
+
+    store = hass.data[DATA_TRACE_STORE]
+    try:
+        restored_traces = await store.async_load() or {}
+    except HomeAssistantError:
+        _LOGGER.exception("Error loading traces")
+        restored_traces = {}
+
+    for key, traces in restored_traces.items():
+        # Add stored traces in reversed order to priorize the newest traces
+        for json_trace in reversed(traces):
+            if (
+                (stored_traces := hass.data[DATA_TRACE].get(key))
+                and stored_traces.size_limit is not None
+                and len(stored_traces) >= stored_traces.size_limit
+            ):
+                break
+
+            try:
+                trace = RestoredTrace(json_trace)
+            # Catch any exception to not blow up if the stored trace is invalid
+            except Exception:  # pylint: disable=broad-except
+                _LOGGER.exception("Failed to restore trace")
+                continue
+            _async_store_restored_trace(hass, trace)
+
+
+class BaseTrace(abc.ABC):
     """Base container for a script or automation trace."""
 
-    _run_ids = count(0)
+    context: Context
+    key: str
+
+    def as_dict(self) -> dict[str, Any]:
+        """Return an dictionary version of this ActionTrace for saving."""
+        return {
+            "extended_dict": self.as_extended_dict(),
+            "short_dict": self.as_short_dict(),
+        }
+
+    @abc.abstractmethod
+    def as_extended_dict(self) -> dict[str, Any]:
+        """Return an extended dictionary version of this ActionTrace."""
+
+    @abc.abstractmethod
+    def as_short_dict(self) -> dict[str, Any]:
+        """Return a brief dictionary version of this ActionTrace."""
+
+
+class ActionTrace(BaseTrace):
+    """Base container for a script or automation trace."""
+
+    _domain: str | None = None
 
     def __init__(
         self,
-        key: tuple[str, str],
+        item_id: str,
         config: dict[str, Any],
         blueprint_inputs: dict[str, Any],
         context: Context,
@@ -69,16 +227,18 @@ class ActionTrace:
         self._error: Exception | None = None
         self._state: str = "running"
         self._script_execution: str | None = None
-        self.run_id: str = str(next(self._run_ids))
+        self.run_id: str = uuid_util.random_uuid_hex()
         self._timestamp_finish: dt.datetime | None = None
         self._timestamp_start: dt.datetime = dt_util.utcnow()
-        self.key: tuple[str, str] = key
+        self.key = f"{self._domain}.{item_id}"
+        self._dict: dict[str, Any] | None = None
+        self._short_dict: dict[str, Any] | None = None
         if trace_id_get():
             trace_set_child_id(self.key, self.run_id)
-        trace_id_set((key, self.run_id))
+        trace_id_set((self.key, self.run_id))
 
     def set_trace(self, trace: dict[str, deque[TraceElement]]) -> None:
-        """Set trace."""
+        """Set action trace."""
         self._trace = trace
 
     def set_error(self, ex: Exception) -> None:
@@ -91,10 +251,12 @@ class ActionTrace:
         self._state = "stopped"
         self._script_execution = script_execution_get()
 
-    def as_dict(self) -> dict[str, Any]:
+    def as_extended_dict(self) -> dict[str, Any]:
-        """Return dictionary version of this ActionTrace."""
+        """Return an extended dictionary version of this ActionTrace."""
+        if self._dict:
+            return self._dict
 
-        result = self.as_short_dict()
+        result = dict(self.as_short_dict())
 
         traces = {}
         if self._trace:
@@ -110,15 +272,21 @@ class ActionTrace:
             }
         )
 
+        if self._state == "stopped":
+            # Execution has stopped, save the result
+            self._dict = result
         return result
 
     def as_short_dict(self) -> dict[str, Any]:
         """Return a brief dictionary version of this ActionTrace."""
+        if self._short_dict:
+            return self._short_dict
 
         last_step = None
 
         if self._trace:
             last_step = list(self._trace)[-1]
+        domain, item_id = self.key.split(".", 1)
 
         result = {
             "last_step": last_step,
@@ -129,10 +297,40 @@ class ActionTrace:
                 "start": self._timestamp_start,
                 "finish": self._timestamp_finish,
             },
-            "domain": self.key[0],
+            "domain": domain,
-            "item_id": self.key[1],
+            "item_id": item_id,
         }
         if self._error is not None:
             result["error"] = str(self._error)
 
+        if self._state == "stopped":
+            # Execution has stopped, save the result
+            self._short_dict = result
         return result
+
+
+class RestoredTrace(BaseTrace):
+    """Container for a restored script or automation trace."""
+
+    def __init__(self, data: dict[str, Any]) -> None:
+        """Restore from dict."""
+        extended_dict = data["extended_dict"]
+        short_dict = data["short_dict"]
+        context = Context(
+            user_id=extended_dict["context"]["user_id"],
+            parent_id=extended_dict["context"]["parent_id"],
+            id=extended_dict["context"]["id"],
+        )
+        self.context = context
+        self.key = f"{extended_dict['domain']}.{extended_dict['item_id']}"
+        self.run_id = extended_dict["run_id"]
+        self._dict = extended_dict
+        self._short_dict = short_dict
+
+    def as_extended_dict(self) -> dict[str, Any]:
+        """Return an extended dictionary version of this RestoredTrace."""
+        return self._dict
+
+    def as_short_dict(self) -> dict[str, Any]:
+        """Return a brief dictionary version of this RestoredTrace."""
+        return self._short_dict

homeassistant/components/trace/const.py

@@ -2,4 +2,6 @@
 
 CONF_STORED_TRACES = "stored_traces"
 DATA_TRACE = "trace"
+DATA_TRACE_STORE = "trace_store"
+DATA_TRACES_RESTORED = "trace_traces_restored"
 DEFAULT_STORED_TRACES = 5  # Stored traces per script or automation

homeassistant/components/trace/websocket_api.py

@@ -3,7 +3,7 @@ import json
 
 import voluptuous as vol
 
-from homeassistant.components import websocket_api
+from homeassistant.components import trace, websocket_api
 from homeassistant.core import HomeAssistant, callback
 from homeassistant.exceptions import HomeAssistantError
 from homeassistant.helpers.dispatcher import (
@@ -24,8 +24,6 @@ from homeassistant.helpers.script import (
     debug_stop,
 )
 
-from .const import DATA_TRACE
-
 # mypy: allow-untyped-calls, allow-untyped-defs
 
 TRACE_DOMAINS = ("automation", "script")
@@ -46,7 +44,6 @@ def async_setup(hass: HomeAssistant) -> None:
     websocket_api.async_register_command(hass, websocket_subscribe_breakpoint_events)
 
 
-@callback
 @websocket_api.require_admin
 @websocket_api.websocket_command(
     {
@@ -56,37 +53,27 @@ def async_setup(hass: HomeAssistant) -> None:
         vol.Required("run_id"): str,
     }
 )
-def websocket_trace_get(hass, connection, msg):
+@websocket_api.async_response
+async def websocket_trace_get(hass, connection, msg):
     """Get a script or automation trace."""
-    key = (msg["domain"], msg["item_id"])
+    key = f"{msg['domain']}.{msg['item_id']}"
     run_id = msg["run_id"]
 
     try:
-        trace = hass.data[DATA_TRACE][key][run_id]
+        requested_trace = await trace.async_get_trace(hass, key, run_id)
     except KeyError:
         connection.send_error(
             msg["id"], websocket_api.ERR_NOT_FOUND, "The trace could not be found"
         )
         return
 
-    message = websocket_api.messages.result_message(msg["id"], trace)
+    message = websocket_api.messages.result_message(msg["id"], requested_trace)
 
     connection.send_message(
         json.dumps(message, cls=ExtendedJSONEncoder, allow_nan=False)
     )
 
 
-def get_debug_traces(hass, key):
-    """Return a serializable list of debug traces for a script or automation."""
-    traces = []
-
-    for trace in hass.data[DATA_TRACE].get(key, {}).values():
-        traces.append(trace.as_short_dict())
-
-    return traces
-
-
-@callback
 @websocket_api.require_admin
 @websocket_api.websocket_command(
     {
@@ -95,23 +82,17 @@ def get_debug_traces(hass, key):
         vol.Optional("item_id", "id"): str,
     }
 )
-def websocket_trace_list(hass, connection, msg):
+@websocket_api.async_response
+async def websocket_trace_list(hass, connection, msg):
     """Summarize script and automation traces."""
-    domain = msg["domain"]
+    wanted_domain = msg["domain"]
-    key = (domain, msg["item_id"]) if "item_id" in msg else None
+    key = f"{msg['domain']}.{msg['item_id']}" if "item_id" in msg else None
 
-    if not key:
+    traces = await trace.async_list_traces(hass, wanted_domain, key)
-        traces = []
-        for key in hass.data[DATA_TRACE]:
-            if key[0] == domain:
-                traces.extend(get_debug_traces(hass, key))
-    else:
-        traces = get_debug_traces(hass, key)
 
     connection.send_result(msg["id"], traces)
 
 
-@callback
 @websocket_api.require_admin
 @websocket_api.websocket_command(
     {
@@ -120,20 +101,12 @@ def websocket_trace_list(hass, connection, msg):
         vol.Inclusive("item_id", "id"): str,
     }
 )
-def websocket_trace_contexts(hass, connection, msg):
+@websocket_api.async_response
+async def websocket_trace_contexts(hass, connection, msg):
     """Retrieve contexts we have traces for."""
-    key = (msg["domain"], msg["item_id"]) if "item_id" in msg else None
+    key = f"{msg['domain']}.{msg['item_id']}" if "item_id" in msg else None
 
-    if key is not None:
+    contexts = await trace.async_list_contexts(hass, key)
-        values = {key: hass.data[DATA_TRACE].get(key, {})}
-    else:
-        values = hass.data[DATA_TRACE]
-
-    contexts = {
-        trace.context.id: {"run_id": trace.run_id, "domain": key[0], "item_id": key[1]}
-        for key, traces in values.items()
-        for trace in traces.values()
-    }
 
     connection.send_result(msg["id"], contexts)
 
@@ -151,7 +124,7 @@ def websocket_trace_contexts(hass, connection, msg):
 )
 def websocket_breakpoint_set(hass, connection, msg):
     """Set breakpoint."""
-    key = (msg["domain"], msg["item_id"])
+    key = f"{msg['domain']}.{msg['item_id']}"
     node = msg["node"]
     run_id = msg.get("run_id")
 
@@ -178,7 +151,7 @@ def websocket_breakpoint_set(hass, connection, msg):
 )
 def websocket_breakpoint_clear(hass, connection, msg):
     """Clear breakpoint."""
-    key = (msg["domain"], msg["item_id"])
+    key = f"{msg['domain']}.{msg['item_id']}"
     node = msg["node"]
     run_id = msg.get("run_id")
 
@@ -194,7 +167,8 @@ def websocket_breakpoint_list(hass, connection, msg):
     """List breakpoints."""
     breakpoints = breakpoint_list(hass)
     for _breakpoint in breakpoints:
-        _breakpoint["domain"], _breakpoint["item_id"] = _breakpoint.pop("key")
+        key = _breakpoint.pop("key")
+        _breakpoint["domain"], _breakpoint["item_id"] = key.split(".", 1)
 
     connection.send_result(msg["id"], breakpoints)
 
@@ -210,12 +184,13 @@ def websocket_subscribe_breakpoint_events(hass, connection, msg):
     @callback
     def breakpoint_hit(key, run_id, node):
         """Forward events to websocket."""
+        domain, item_id = key.split(".", 1)
         connection.send_message(
             websocket_api.event_message(
                 msg["id"],
                 {
-                    "domain": key[0],
+                    "domain": domain,
-                    "item_id": key[1],
+                    "item_id": item_id,
                     "run_id": run_id,
                     "node": node,
                 },
@@ -254,7 +229,7 @@ def websocket_subscribe_breakpoint_events(hass, connection, msg):
 )
 def websocket_debug_continue(hass, connection, msg):
     """Resume execution of halted script or automation."""
-    key = (msg["domain"], msg["item_id"])
+    key = f"{msg['domain']}.{msg['item_id']}"
     run_id = msg["run_id"]
 
     result = debug_continue(hass, key, run_id)
@@ -274,7 +249,7 @@ def websocket_debug_continue(hass, connection, msg):
 )
 def websocket_debug_step(hass, connection, msg):
     """Single step a halted script or automation."""
-    key = (msg["domain"], msg["item_id"])
+    key = f"{msg['domain']}.{msg['item_id']}"
     run_id = msg["run_id"]
 
     result = debug_step(hass, key, run_id)
@@ -294,7 +269,7 @@ def websocket_debug_step(hass, connection, msg):
 )
 def websocket_debug_stop(hass, connection, msg):
     """Stop a halted script or automation."""
-    key = (msg["domain"], msg["item_id"])
+    key = f"{msg['domain']}.{msg['item_id']}"
     run_id = msg["run_id"]
 
     result = debug_stop(hass, key, run_id)

homeassistant/helpers/trace.py

@@ -17,7 +17,7 @@ class TraceElement:
 
     def __init__(self, variables: TemplateVarsType, path: str) -> None:
         """Container for trace data."""
-        self._child_key: tuple[str, str] | None = None
+        self._child_key: str | None = None
         self._child_run_id: str | None = None
         self._error: Exception | None = None
         self.path: str = path
@@ -40,7 +40,7 @@ class TraceElement:
         """Container for trace data."""
         return str(self.as_dict())
 
-    def set_child_id(self, child_key: tuple[str, str], child_run_id: str) -> None:
+    def set_child_id(self, child_key: str, child_run_id: str) -> None:
         """Set trace id of a nested script run."""
         self._child_key = child_key
         self._child_run_id = child_run_id
@@ -62,9 +62,10 @@ class TraceElement:
         """Return dictionary version of this TraceElement."""
         result: dict[str, Any] = {"path": self.path, "timestamp": self._timestamp}
         if self._child_key is not None:
+            domain, item_id = self._child_key.split(".", 1)
             result["child_id"] = {
-                "domain": self._child_key[0],
+                "domain": domain,
-                "item_id": self._child_key[1],
+                "item_id": item_id,
                 "run_id": str(self._child_run_id),
             }
         if self._variables:
@@ -91,8 +92,8 @@ trace_path_stack_cv: ContextVar[list[str] | None] = ContextVar(
 )
 # Copy of last variables
 variables_cv: ContextVar[Any | None] = ContextVar("variables_cv", default=None)
-# (domain, item_id) + Run ID
+# (domain.item_id, Run ID)
-trace_id_cv: ContextVar[tuple[tuple[str, str], str] | None] = ContextVar(
+trace_id_cv: ContextVar[tuple[str, str] | None] = ContextVar(
     "trace_id_cv", default=None
 )
 # Reason for stopped script execution
@@ -101,12 +102,12 @@ script_execution_cv: ContextVar[StopReason | None] = ContextVar(
 )
 
 
-def trace_id_set(trace_id: tuple[tuple[str, str], str]) -> None:
+def trace_id_set(trace_id: tuple[str, str]) -> None:
     """Set id of the current trace."""
     trace_id_cv.set(trace_id)
 
 
-def trace_id_get() -> tuple[tuple[str, str], str] | None:
+def trace_id_get() -> tuple[str, str] | None:
     """Get id if the current trace."""
     return trace_id_cv.get()
 
@@ -182,7 +183,7 @@ def trace_clear() -> None:
     script_execution_cv.set(StopReason())
 
 
-def trace_set_child_id(child_key: tuple[str, str], child_run_id: str) -> None:
+def trace_set_child_id(child_key: str, child_run_id: str) -> None:
     """Set child trace_id of TraceElement at the top of the stack."""
     node = cast(TraceElement, trace_stack_top(trace_stack_cv))
     if node:

tests/components/script/test_init.py

@@ -1,7 +1,6 @@
 """The tests for the Script component."""
 # pylint: disable=protected-access
 import asyncio
-import unittest
 from unittest.mock import Mock, patch
 
 import pytest
@@ -29,113 +28,62 @@ from homeassistant.exceptions import ServiceNotFound
 from homeassistant.helpers import template
 from homeassistant.helpers.event import async_track_state_change
 from homeassistant.helpers.service import async_get_all_descriptions
-from homeassistant.loader import bind_hass
+from homeassistant.setup import async_setup_component
-from homeassistant.setup import async_setup_component, setup_component
 import homeassistant.util.dt as dt_util
 
-from tests.common import async_mock_service, get_test_home_assistant, mock_restore_cache
+from tests.common import async_mock_service, mock_restore_cache
 from tests.components.logbook.test_init import MockLazyEventPartialState
 
 ENTITY_ID = "script.test"
 
 
-@bind_hass
+async def test_passing_variables(hass):
-def turn_on(hass, entity_id, variables=None, context=None):
+    """Test different ways of passing in variables."""
-    """Turn script on.
+    mock_restore_cache(hass, ())
+    calls = []
+    context = Context()
 
-    This is a legacy helper method. Do not use it for new tests.
+    @callback
-    """
+    def record_call(service):
-    _, object_id = split_entity_id(entity_id)
+        """Add recorded event to set."""
+        calls.append(service)
 
-    hass.services.call(DOMAIN, object_id, variables, context=context)
+    hass.services.async_register("test", "script", record_call)
 
-
+    assert await async_setup_component(
-@bind_hass
+        hass,
-def turn_off(hass, entity_id):
+        "script",
-    """Turn script on.
+        {
-
+            "script": {
-    This is a legacy helper method. Do not use it for new tests.
+                "test": {
-    """
+                    "sequence": {
-    hass.services.call(DOMAIN, SERVICE_TURN_OFF, {ATTR_ENTITY_ID: entity_id})
+                        "service": "test.script",
-
+                        "data_template": {"hello": "{{ greeting }}"},
-
-@bind_hass
-def toggle(hass, entity_id):
-    """Toggle the script.
-
-    This is a legacy helper method. Do not use it for new tests.
-    """
-    hass.services.call(DOMAIN, SERVICE_TOGGLE, {ATTR_ENTITY_ID: entity_id})
-
-
-@bind_hass
-def reload(hass):
-    """Reload script component.
-
-    This is a legacy helper method. Do not use it for new tests.
-    """
-    hass.services.call(DOMAIN, SERVICE_RELOAD)
-
-
-class TestScriptComponent(unittest.TestCase):
-    """Test the Script component."""
-
-    # pylint: disable=invalid-name
-    def setUp(self):
-        """Set up things to be run when tests are started."""
-        self.hass = get_test_home_assistant()
-
-        self.addCleanup(self.tear_down_cleanup)
-
-    def tear_down_cleanup(self):
-        """Stop down everything that was started."""
-        self.hass.stop()
-
-    def test_passing_variables(self):
-        """Test different ways of passing in variables."""
-        mock_restore_cache(self.hass, ())
-        calls = []
-        context = Context()
-
-        @callback
-        def record_call(service):
-            """Add recorded event to set."""
-            calls.append(service)
-
-        self.hass.services.register("test", "script", record_call)
-
-        assert setup_component(
-            self.hass,
-            "script",
-            {
-                "script": {
-                    "test": {
-                        "sequence": {
-                            "service": "test.script",
-                            "data_template": {"hello": "{{ greeting }}"},
-                        }
                     }
                 }
-            },
+            }
-        )
+        },
+    )
 
-        turn_on(self.hass, ENTITY_ID, {"greeting": "world"}, context=context)
+    await hass.services.async_call(
+        DOMAIN, "test", {"greeting": "world"}, context=context
+    )
 
-        self.hass.block_till_done()
+    await hass.async_block_till_done()
 
-        assert len(calls) == 1
+    assert len(calls) == 1
-        assert calls[0].context is context
+    assert calls[0].context is context
-        assert calls[0].data["hello"] == "world"
+    assert calls[0].data["hello"] == "world"
 
-        self.hass.services.call(
+    await hass.services.async_call(
-            "script", "test", {"greeting": "universe"}, context=context
+        "script", "test", {"greeting": "universe"}, context=context
-        )
+    )
 
-        self.hass.block_till_done()
+    await hass.async_block_till_done()
 
-        assert len(calls) == 2
+    assert len(calls) == 2
-        assert calls[1].context is context
+    assert calls[1].context is context
-        assert calls[1].data["hello"] == "universe"
+    assert calls[1].data["hello"] == "universe"
 
 
 @pytest.mark.parametrize("toggle", [False, True])

tests/components/trace/test_websocket_api.py

@@ -1,14 +1,19 @@
 """Test Trace websocket API."""
 import asyncio
+import json
+from typing import DefaultDict
+from unittest.mock import patch
 
 import pytest
 
 from homeassistant.bootstrap import async_setup_component
 from homeassistant.components.trace.const import DEFAULT_STORED_TRACES
-from homeassistant.core import Context, callback
+from homeassistant.const import EVENT_HOMEASSISTANT_STOP
+from homeassistant.core import Context, CoreState, callback
 from homeassistant.helpers.typing import UNDEFINED
+from homeassistant.util.uuid import random_uuid_hex
 
-from tests.common import assert_lists_same
+from tests.common import assert_lists_same, load_fixture
 
 
 def _find_run_id(traces, trace_type, item_id):
@@ -70,8 +75,12 @@ def _assert_raw_config(domain, config, trace):
     assert trace["config"] == config
 
 
-async def _assert_contexts(client, next_id, contexts):
+async def _assert_contexts(client, next_id, contexts, domain=None, item_id=None):
-    await client.send_json({"id": next_id(), "type": "trace/contexts"})
+    request = {"id": next_id(), "type": "trace/contexts"}
+    if domain is not None:
+        request["domain"] = domain
+        request["item_id"] = item_id
+    await client.send_json(request)
     response = await client.receive_json()
     assert response["success"]
     assert response["result"] == contexts
@@ -101,6 +110,7 @@ async def _assert_contexts(client, next_id, contexts):
 )
 async def test_get_trace(
     hass,
+    hass_storage,
     hass_ws_client,
     domain,
     prefix,
@@ -152,6 +162,8 @@ async def test_get_trace(
 
     client = await hass_ws_client()
     contexts = {}
+    contexts_sun = {}
+    contexts_moon = {}
 
     # Trigger "sun" automation / run "sun" script
     context = Context()
@@ -195,6 +207,11 @@ async def test_get_trace(
         "domain": domain,
         "item_id": trace["item_id"],
     }
+    contexts_sun[trace["context"]["id"]] = {
+        "run_id": trace["run_id"],
+        "domain": domain,
+        "item_id": trace["item_id"],
+    }
 
     # Trigger "moon" automation, with passing condition / run "moon" script
     await _run_automation_or_script(hass, domain, moon_config, "test_event2", context)
@@ -244,10 +261,17 @@ async def test_get_trace(
         "domain": domain,
         "item_id": trace["item_id"],
     }
+    contexts_moon[trace["context"]["id"]] = {
+        "run_id": trace["run_id"],
+        "domain": domain,
+        "item_id": trace["item_id"],
+    }
 
     if len(extra_trace_keys) <= 2:
         # Check contexts
         await _assert_contexts(client, next_id, contexts)
+        await _assert_contexts(client, next_id, contexts_moon, domain, "moon")
+        await _assert_contexts(client, next_id, contexts_sun, domain, "sun")
         return
 
     # Trigger "moon" automation with failing condition
@@ -291,6 +315,11 @@ async def test_get_trace(
         "domain": domain,
         "item_id": trace["item_id"],
     }
+    contexts_moon[trace["context"]["id"]] = {
+        "run_id": trace["run_id"],
+        "domain": domain,
+        "item_id": trace["item_id"],
+    }
 
     # Trigger "moon" automation with passing condition
     hass.bus.async_fire("test_event2")
@@ -336,9 +365,119 @@ async def test_get_trace(
         "domain": domain,
         "item_id": trace["item_id"],
     }
+    contexts_moon[trace["context"]["id"]] = {
+        "run_id": trace["run_id"],
+        "domain": domain,
+        "item_id": trace["item_id"],
+    }
 
     # Check contexts
     await _assert_contexts(client, next_id, contexts)
+    await _assert_contexts(client, next_id, contexts_moon, domain, "moon")
+    await _assert_contexts(client, next_id, contexts_sun, domain, "sun")
+
+    # List traces
+    await client.send_json({"id": next_id(), "type": "trace/list", "domain": domain})
+    response = await client.receive_json()
+    assert response["success"]
+    trace_list = response["result"]
+
+    # Get all traces and generate expected stored traces
+    traces = DefaultDict(list)
+    for trace in trace_list:
+        item_id = trace["item_id"]
+        run_id = trace["run_id"]
+        await client.send_json(
+            {
+                "id": next_id(),
+                "type": "trace/get",
+                "domain": domain,
+                "item_id": item_id,
+                "run_id": run_id,
+            }
+        )
+        response = await client.receive_json()
+        assert response["success"]
+        traces[f"{domain}.{item_id}"].append(
+            {"short_dict": trace, "extended_dict": response["result"]}
+        )
+
+    # Fake stop
+    assert "trace.saved_traces" not in hass_storage
+    hass.bus.async_fire(EVENT_HOMEASSISTANT_STOP)
+    await hass.async_block_till_done()
+
+    # Check that saved data is same as the serialized traces
+    assert "trace.saved_traces" in hass_storage
+    assert hass_storage["trace.saved_traces"]["data"] == traces
+
+
+@pytest.mark.parametrize("domain", ["automation", "script"])
+async def test_restore_traces(hass, hass_storage, hass_ws_client, domain):
+    """Test restored traces."""
+    hass.state = CoreState.not_running
+    id = 1
+
+    def next_id():
+        nonlocal id
+        id += 1
+        return id
+
+    saved_traces = json.loads(load_fixture(f"trace/{domain}_saved_traces.json"))
+    hass_storage["trace.saved_traces"] = saved_traces
+    await _setup_automation_or_script(hass, domain, [])
+    await hass.async_start()
+    await hass.async_block_till_done()
+
+    client = await hass_ws_client()
+
+    # List traces
+    await client.send_json({"id": next_id(), "type": "trace/list", "domain": domain})
+    response = await client.receive_json()
+    assert response["success"]
+    trace_list = response["result"]
+
+    # Get all traces and generate expected stored traces
+    traces = DefaultDict(list)
+    contexts = {}
+    for trace in trace_list:
+        item_id = trace["item_id"]
+        run_id = trace["run_id"]
+        await client.send_json(
+            {
+                "id": next_id(),
+                "type": "trace/get",
+                "domain": domain,
+                "item_id": item_id,
+                "run_id": run_id,
+            }
+        )
+        response = await client.receive_json()
+        assert response["success"]
+        traces[f"{domain}.{item_id}"].append(
+            {"short_dict": trace, "extended_dict": response["result"]}
+        )
+        contexts[response["result"]["context"]["id"]] = {
+            "run_id": trace["run_id"],
+            "domain": domain,
+            "item_id": trace["item_id"],
+        }
+
+    # Check that loaded data is same as the serialized traces
+    assert hass_storage["trace.saved_traces"]["data"] == traces
+
+    # Check restored contexts
+    await _assert_contexts(client, next_id, contexts)
+
+    # Fake stop
+    hass_storage.pop("trace.saved_traces")
+    assert "trace.saved_traces" not in hass_storage
+    hass.bus.async_fire(EVENT_HOMEASSISTANT_STOP)
+    await hass.async_block_till_done()
+
+    # Check that saved data is same as the serialized traces
+    assert "trace.saved_traces" in hass_storage
+    assert hass_storage["trace.saved_traces"] == saved_traces
 
 
 @pytest.mark.parametrize("domain", ["automation", "script"])
@@ -368,6 +507,13 @@ async def test_trace_overflow(hass, hass_ws_client, domain, stored_traces):
     """Test the number of stored traces per script or automation is limited."""
     id = 1
 
+    trace_uuids = []
+
+    def mock_random_uuid_hex():
+        nonlocal trace_uuids
+        trace_uuids.append(random_uuid_hex())
+        return trace_uuids[-1]
+
     def next_id():
         nonlocal id
         id += 1
@@ -404,13 +550,16 @@ async def test_trace_overflow(hass, hass_ws_client, domain, stored_traces):
     response = await client.receive_json()
     assert response["success"]
     assert len(_find_traces(response["result"], domain, "moon")) == 1
-    moon_run_id = _find_run_id(response["result"], domain, "moon")
     assert len(_find_traces(response["result"], domain, "sun")) == 1
 
     # Trigger "moon" enough times to overflow the max number of stored traces
-    for _ in range(stored_traces or DEFAULT_STORED_TRACES):
+    with patch(
-        await _run_automation_or_script(hass, domain, moon_config, "test_event2")
+        "homeassistant.components.trace.uuid_util.random_uuid_hex",
-        await hass.async_block_till_done()
+        wraps=mock_random_uuid_hex,
+    ):
+        for _ in range(stored_traces or DEFAULT_STORED_TRACES):
+            await _run_automation_or_script(hass, domain, moon_config, "test_event2")
+            await hass.async_block_till_done()
 
     await client.send_json({"id": next_id(), "type": "trace/list", "domain": domain})
     response = await client.receive_json()
@@ -418,10 +567,153 @@ async def test_trace_overflow(hass, hass_ws_client, domain, stored_traces):
     moon_traces = _find_traces(response["result"], domain, "moon")
     assert len(moon_traces) == stored_traces or DEFAULT_STORED_TRACES
     assert moon_traces[0]
-    assert int(moon_traces[0]["run_id"]) == int(moon_run_id) + 1
+    assert moon_traces[0]["run_id"] == trace_uuids[0]
-    assert int(moon_traces[-1]["run_id"]) == int(moon_run_id) + (
+    assert moon_traces[-1]["run_id"] == trace_uuids[-1]
-        stored_traces or DEFAULT_STORED_TRACES
+    assert len(_find_traces(response["result"], domain, "sun")) == 1
-    )
+
+
+@pytest.mark.parametrize(
+    "domain,num_restored_moon_traces", [("automation", 3), ("script", 1)]
+)
+async def test_restore_traces_overflow(
+    hass, hass_storage, hass_ws_client, domain, num_restored_moon_traces
+):
+    """Test restored traces are evicted first."""
+    hass.state = CoreState.not_running
+    id = 1
+
+    trace_uuids = []
+
+    def mock_random_uuid_hex():
+        nonlocal trace_uuids
+        trace_uuids.append(random_uuid_hex())
+        return trace_uuids[-1]
+
+    def next_id():
+        nonlocal id
+        id += 1
+        return id
+
+    saved_traces = json.loads(load_fixture(f"trace/{domain}_saved_traces.json"))
+    hass_storage["trace.saved_traces"] = saved_traces
+    sun_config = {
+        "id": "sun",
+        "trigger": {"platform": "event", "event_type": "test_event"},
+        "action": {"event": "some_event"},
+    }
+    moon_config = {
+        "id": "moon",
+        "trigger": {"platform": "event", "event_type": "test_event2"},
+        "action": {"event": "another_event"},
+    }
+    await _setup_automation_or_script(hass, domain, [sun_config, moon_config])
+    await hass.async_start()
+    await hass.async_block_till_done()
+
+    client = await hass_ws_client()
+
+    # Traces should not yet be restored
+    assert "trace_traces_restored" not in hass.data
+
+    # List traces
+    await client.send_json({"id": next_id(), "type": "trace/list", "domain": domain})
+    response = await client.receive_json()
+    assert response["success"]
+    restored_moon_traces = _find_traces(response["result"], domain, "moon")
+    assert len(restored_moon_traces) == num_restored_moon_traces
+    assert len(_find_traces(response["result"], domain, "sun")) == 1
+
+    # Traces should be restored
+    assert "trace_traces_restored" in hass.data
+
+    # Trigger "moon" enough times to overflow the max number of stored traces
+    with patch(
+        "homeassistant.components.trace.uuid_util.random_uuid_hex",
+        wraps=mock_random_uuid_hex,
+    ):
+        for _ in range(DEFAULT_STORED_TRACES - num_restored_moon_traces + 1):
+            await _run_automation_or_script(hass, domain, moon_config, "test_event2")
+            await hass.async_block_till_done()
+
+    await client.send_json({"id": next_id(), "type": "trace/list", "domain": domain})
+    response = await client.receive_json()
+    assert response["success"]
+    moon_traces = _find_traces(response["result"], domain, "moon")
+    assert len(moon_traces) == DEFAULT_STORED_TRACES
+    if num_restored_moon_traces > 1:
+        assert moon_traces[0]["run_id"] == restored_moon_traces[1]["run_id"]
+    assert moon_traces[num_restored_moon_traces - 1]["run_id"] == trace_uuids[0]
+    assert moon_traces[-1]["run_id"] == trace_uuids[-1]
+    assert len(_find_traces(response["result"], domain, "sun")) == 1
+
+
+@pytest.mark.parametrize(
+    "domain,num_restored_moon_traces,restored_run_id",
+    [("automation", 3, "e2c97432afe9b8a42d7983588ed5e6ef"), ("script", 1, "")],
+)
+async def test_restore_traces_late_overflow(
+    hass,
+    hass_storage,
+    hass_ws_client,
+    domain,
+    num_restored_moon_traces,
+    restored_run_id,
+):
+    """Test restored traces are evicted first."""
+    hass.state = CoreState.not_running
+    id = 1
+
+    trace_uuids = []
+
+    def mock_random_uuid_hex():
+        nonlocal trace_uuids
+        trace_uuids.append(random_uuid_hex())
+        return trace_uuids[-1]
+
+    def next_id():
+        nonlocal id
+        id += 1
+        return id
+
+    saved_traces = json.loads(load_fixture(f"trace/{domain}_saved_traces.json"))
+    hass_storage["trace.saved_traces"] = saved_traces
+    sun_config = {
+        "id": "sun",
+        "trigger": {"platform": "event", "event_type": "test_event"},
+        "action": {"event": "some_event"},
+    }
+    moon_config = {
+        "id": "moon",
+        "trigger": {"platform": "event", "event_type": "test_event2"},
+        "action": {"event": "another_event"},
+    }
+    await _setup_automation_or_script(hass, domain, [sun_config, moon_config])
+    await hass.async_start()
+    await hass.async_block_till_done()
+
+    client = await hass_ws_client()
+
+    # Traces should not yet be restored
+    assert "trace_traces_restored" not in hass.data
+
+    # Trigger "moon" enough times to overflow the max number of stored traces
+    with patch(
+        "homeassistant.components.trace.uuid_util.random_uuid_hex",
+        wraps=mock_random_uuid_hex,
+    ):
+        for _ in range(DEFAULT_STORED_TRACES - num_restored_moon_traces + 1):
+            await _run_automation_or_script(hass, domain, moon_config, "test_event2")
+            await hass.async_block_till_done()
+
+    await client.send_json({"id": next_id(), "type": "trace/list", "domain": domain})
+    response = await client.receive_json()
+    assert response["success"]
+    moon_traces = _find_traces(response["result"], domain, "moon")
+    assert len(moon_traces) == DEFAULT_STORED_TRACES
+    if num_restored_moon_traces > 1:
+        assert moon_traces[0]["run_id"] == restored_run_id
+    assert moon_traces[num_restored_moon_traces - 1]["run_id"] == trace_uuids[0]
+    assert moon_traces[-1]["run_id"] == trace_uuids[-1]
     assert len(_find_traces(response["result"], domain, "sun")) == 1
 
 

tests/fixtures/trace/automation_saved_traces.json

@@ -0,0 +1,486 @@
+{
+    "version": 1,
+    "key": "trace.saved_traces",
+    "data": {
+        "automation.sun": [
+            {
+                "extended_dict": {
+                    "last_step": "action/0",
+                    "run_id": "d09f46a4007732c53fa69f434acc1c02",
+                    "state": "stopped",
+                    "script_execution": "error",
+                    "timestamp": {
+                        "start": "2021-10-14T06:43:39.540977+00:00",
+                        "finish": "2021-10-14T06:43:39.542744+00:00"
+                    },
+                    "domain": "automation",
+                    "item_id": "sun",
+                    "error": "Unable to find service test.automation",
+                    "trigger": "event 'test_event'",
+                    "trace": {
+                        "trigger/0": [
+                            {
+                                "path": "trigger/0",
+                                "timestamp": "2021-10-14T06:43:39.541024+00:00",
+                                "changed_variables": {
+                                    "this": {
+                                        "entity_id": "automation.automation_0",
+                                        "state": "on",
+                                        "attributes": {
+                                            "last_triggered": null,
+                                            "mode": "single",
+                                            "current": 0,
+                                            "id": "sun",
+                                            "friendly_name": "automation 0"
+                                        },
+                                        "last_changed": "2021-10-14T06:43:39.368423+00:00",
+                                        "last_updated": "2021-10-14T06:43:39.368423+00:00",
+                                        "context": {
+                                            "id": "c62f6b3f975b4f9bd479b10a4d7425db",
+                                            "parent_id": null,
+                                            "user_id": null
+                                        }
+                                    },
+                                    "trigger": {
+                                        "id": "0",
+                                        "idx": "0",
+                                        "platform": "event",
+                                        "event": {
+                                            "event_type": "test_event",
+                                            "data": {},
+                                            "origin": "LOCAL",
+                                            "time_fired": "2021-10-14T06:43:39.540382+00:00",
+                                            "context": {
+                                                "id": "66934a357e691e845d7f00ee953c0f0f",
+                                                "parent_id": null,
+                                                "user_id": null
+                                            }
+                                        },
+                                        "description": "event 'test_event'"
+                                    }
+                                }
+                            }
+                        ],
+                        "action/0": [
+                            {
+                                "path": "action/0",
+                                "timestamp": "2021-10-14T06:43:39.541738+00:00",
+                                "changed_variables": {
+                                    "context": {
+                                        "id": "4438e85e335bd05e6474d2846d7001cc",
+                                        "parent_id": "66934a357e691e845d7f00ee953c0f0f",
+                                        "user_id": null
+                                    }
+                                },
+                                "error": "Unable to find service test.automation",
+                                "result": {
+                                    "params": {
+                                        "domain": "test",
+                                        "service": "automation",
+                                        "service_data": {},
+                                        "target": {}
+                                    },
+                                    "running_script": false,
+                                    "limit": 10
+                                }
+                            }
+                        ]
+                    },
+                    "config": {
+                        "id": "sun",
+                        "trigger": {
+                            "platform": "event",
+                            "event_type": "test_event"
+                        },
+                        "action": {
+                            "service": "test.automation"
+                        }
+                    },
+                    "blueprint_inputs": null,
+                    "context": {
+                        "id": "4438e85e335bd05e6474d2846d7001cc",
+                        "parent_id": "66934a357e691e845d7f00ee953c0f0f",
+                        "user_id": null
+                    }
+                },
+                "short_dict": {
+                    "last_step": "action/0",
+                    "run_id": "d09f46a4007732c53fa69f434acc1c02",
+                    "state": "stopped",
+                    "script_execution": "error",
+                    "timestamp": {
+                        "start": "2021-10-14T06:43:39.540977+00:00",
+                        "finish": "2021-10-14T06:43:39.542744+00:00"
+                    },
+                    "domain": "automation",
+                    "item_id": "sun",
+                    "error": "Unable to find service test.automation",
+                    "trigger": "event 'test_event'"
+                }
+            }
+        ],
+        "automation.moon": [
+            {
+                "extended_dict": {
+                    "last_step": "action/0",
+                    "run_id": "511d210ac62aa04668ab418063b57e2c",
+                    "state": "stopped",
+                    "script_execution": "finished",
+                    "timestamp": {
+                        "start": "2021-10-14T06:43:39.545290+00:00",
+                        "finish": "2021-10-14T06:43:39.546962+00:00"
+                    },
+                    "domain": "automation",
+                    "item_id": "moon",
+                    "trigger": "event 'test_event2'",
+                    "trace": {
+                        "trigger/0": [
+                            {
+                                "path": "trigger/0",
+                                "timestamp": "2021-10-14T06:43:39.545313+00:00",
+                                "changed_variables": {
+                                    "this": {
+                                        "entity_id": "automation.automation_1",
+                                        "state": "on",
+                                        "attributes": {
+                                            "last_triggered": null,
+                                            "mode": "single",
+                                            "current": 0,
+                                            "id": "moon",
+                                            "friendly_name": "automation 1"
+                                        },
+                                        "last_changed": "2021-10-14T06:43:39.369282+00:00",
+                                        "last_updated": "2021-10-14T06:43:39.369282+00:00",
+                                        "context": {
+                                            "id": "c914e818f5b234c0fc0dfddf75e98b0e",
+                                            "parent_id": null,
+                                            "user_id": null
+                                        }
+                                    },
+                                    "trigger": {
+                                        "id": "0",
+                                        "idx": "0",
+                                        "platform": "event",
+                                        "event": {
+                                            "event_type": "test_event2",
+                                            "data": {},
+                                            "origin": "LOCAL",
+                                            "time_fired": "2021-10-14T06:43:39.545003+00:00",
+                                            "context": {
+                                                "id": "66934a357e691e845d7f00ee953c0f0f",
+                                                "parent_id": null,
+                                                "user_id": null
+                                            }
+                                        },
+                                        "description": "event 'test_event2'"
+                                    }
+                                }
+                            }
+                        ],
+                        "condition/0": [
+                            {
+                                "path": "condition/0",
+                                "timestamp": "2021-10-14T06:43:39.545336+00:00",
+                                "result": {
+                                    "result": true,
+                                    "entities": []
+                                }
+                            }
+                        ],
+                        "action/0": [
+                            {
+                                "path": "action/0",
+                                "timestamp": "2021-10-14T06:43:39.546378+00:00",
+                                "changed_variables": {
+                                    "context": {
+                                        "id": "8948898e0074ecaa98be2e041256c81b",
+                                        "parent_id": "66934a357e691e845d7f00ee953c0f0f",
+                                        "user_id": null
+                                    }
+                                },
+                                "result": {
+                                    "event": "another_event",
+                                    "event_data": {}
+                                }
+                            }
+                        ]
+                    },
+                    "config": {
+                        "id": "moon",
+                        "trigger": [
+                            {
+                                "platform": "event",
+                                "event_type": "test_event2"
+                            },
+                            {
+                                "platform": "event",
+                                "event_type": "test_event3"
+                            }
+                        ],
+                        "condition": {
+                            "condition": "template",
+                            "value_template": "{{ trigger.event.event_type=='test_event2' }}"
+                        },
+                        "action": {
+                            "event": "another_event"
+                        }
+                    },
+                    "blueprint_inputs": null,
+                    "context": {
+                        "id": "8948898e0074ecaa98be2e041256c81b",
+                        "parent_id": "66934a357e691e845d7f00ee953c0f0f",
+                        "user_id": null
+                    }
+                },
+                "short_dict": {
+                    "last_step": "action/0",
+                    "run_id": "511d210ac62aa04668ab418063b57e2c",
+                    "state": "stopped",
+                    "script_execution": "finished",
+                    "timestamp": {
+                        "start": "2021-10-14T06:43:39.545290+00:00",
+                        "finish": "2021-10-14T06:43:39.546962+00:00"
+                    },
+                    "domain": "automation",
+                    "item_id": "moon",
+                    "trigger": "event 'test_event2'"
+                }
+            },
+            {
+                "extended_dict": {
+                    "last_step": "condition/0",
+                    "run_id": "e2c97432afe9b8a42d7983588ed5e6ef",
+                    "state": "stopped",
+                    "script_execution": "failed_conditions",
+                    "timestamp": {
+                        "start": "2021-10-14T06:43:39.549081+00:00",
+                        "finish": "2021-10-14T06:43:39.549468+00:00"
+                    },
+                    "domain": "automation",
+                    "item_id": "moon",
+                    "trigger": "event 'test_event3'",
+                    "trace": {
+                        "trigger/1": [
+                            {
+                                "path": "trigger/1",
+                                "timestamp": "2021-10-14T06:43:39.549115+00:00",
+                                "changed_variables": {
+                                    "this": {
+                                        "entity_id": "automation.automation_1",
+                                        "state": "on",
+                                        "attributes": {
+                                            "last_triggered": "2021-10-14T06:43:39.545943+00:00",
+                                            "mode": "single",
+                                            "current": 0,
+                                            "id": "moon",
+                                            "friendly_name": "automation 1"
+                                        },
+                                        "last_changed": "2021-10-14T06:43:39.369282+00:00",
+                                        "last_updated": "2021-10-14T06:43:39.546662+00:00",
+                                        "context": {
+                                            "id": "8948898e0074ecaa98be2e041256c81b",
+                                            "parent_id": "66934a357e691e845d7f00ee953c0f0f",
+                                            "user_id": null
+                                        }
+                                    },
+                                    "trigger": {
+                                        "id": "1",
+                                        "idx": "1",
+                                        "platform": "event",
+                                        "event": {
+                                            "event_type": "test_event3",
+                                            "data": {},
+                                            "origin": "LOCAL",
+                                            "time_fired": "2021-10-14T06:43:39.548788+00:00",
+                                            "context": {
+                                                "id": "5f5113a378b3c06fe146ead2908f6f44",
+                                                "parent_id": null,
+                                                "user_id": null
+                                            }
+                                        },
+                                        "description": "event 'test_event3'"
+                                    }
+                                }
+                            }
+                        ],
+                        "condition/0": [
+                            {
+                                "path": "condition/0",
+                                "timestamp": "2021-10-14T06:43:39.549136+00:00",
+                                "result": {
+                                    "result": false,
+                                    "entities": []
+                                }
+                            }
+                        ]
+                    },
+                    "config": {
+                        "id": "moon",
+                        "trigger": [
+                            {
+                                "platform": "event",
+                                "event_type": "test_event2"
+                            },
+                            {
+                                "platform": "event",
+                                "event_type": "test_event3"
+                            }
+                        ],
+                        "condition": {
+                            "condition": "template",
+                            "value_template": "{{ trigger.event.event_type=='test_event2' }}"
+                        },
+                        "action": {
+                            "event": "another_event"
+                        }
+                    },
+                    "blueprint_inputs": null,
+                    "context": {
+                        "id": "77d041c4e0ecc91ab5e707239c983faf",
+                        "parent_id": "5f5113a378b3c06fe146ead2908f6f44",
+                        "user_id": null
+                    }
+                },
+                "short_dict": {
+                    "last_step": "condition/0",
+                    "run_id": "e2c97432afe9b8a42d7983588ed5e6ef",
+                    "state": "stopped",
+                    "script_execution": "failed_conditions",
+                    "timestamp": {
+                        "start": "2021-10-14T06:43:39.549081+00:00",
+                        "finish": "2021-10-14T06:43:39.549468+00:00"
+                    },
+                    "domain": "automation",
+                    "item_id": "moon",
+                    "trigger": "event 'test_event3'"
+                }
+            },
+            {
+                "extended_dict": {
+                    "last_step": "action/0",
+                    "run_id": "f71d7fa261d361ed999c1dda0a846c99",
+                    "state": "stopped",
+                    "script_execution": "finished",
+                    "timestamp": {
+                        "start": "2021-10-14T06:43:39.551485+00:00",
+                        "finish": "2021-10-14T06:43:39.552822+00:00"
+                    },
+                    "domain": "automation",
+                    "item_id": "moon",
+                    "trigger": "event 'test_event2'",
+                    "trace": {
+                        "trigger/0": [
+                            {
+                                "path": "trigger/0",
+                                "timestamp": "2021-10-14T06:43:39.551503+00:00",
+                                "changed_variables": {
+                                    "this": {
+                                        "entity_id": "automation.automation_1",
+                                        "state": "on",
+                                        "attributes": {
+                                            "last_triggered": "2021-10-14T06:43:39.545943+00:00",
+                                            "mode": "single",
+                                            "current": 0,
+                                            "id": "moon",
+                                            "friendly_name": "automation 1"
+                                        },
+                                        "last_changed": "2021-10-14T06:43:39.369282+00:00",
+                                        "last_updated": "2021-10-14T06:43:39.546662+00:00",
+                                        "context": {
+                                            "id": "8948898e0074ecaa98be2e041256c81b",
+                                            "parent_id": "66934a357e691e845d7f00ee953c0f0f",
+                                            "user_id": null
+                                        }
+                                    },
+                                    "trigger": {
+                                        "id": "0",
+                                        "idx": "0",
+                                        "platform": "event",
+                                        "event": {
+                                            "event_type": "test_event2",
+                                            "data": {},
+                                            "origin": "LOCAL",
+                                            "time_fired": "2021-10-14T06:43:39.551202+00:00",
+                                            "context": {
+                                                "id": "66a59f97502785c544724fdb46bcb94d",
+                                                "parent_id": null,
+                                                "user_id": null
+                                            }
+                                        },
+                                        "description": "event 'test_event2'"
+                                    }
+                                }
+                            }
+                        ],
+                        "condition/0": [
+                            {
+                                "path": "condition/0",
+                                "timestamp": "2021-10-14T06:43:39.551524+00:00",
+                                "result": {
+                                    "result": true,
+                                    "entities": []
+                                }
+                            }
+                        ],
+                        "action/0": [
+                            {
+                                "path": "action/0",
+                                "timestamp": "2021-10-14T06:43:39.552236+00:00",
+                                "changed_variables": {
+                                    "context": {
+                                        "id": "3128b5fa3494cb17cfb485176ef2cee3",
+                                        "parent_id": "66a59f97502785c544724fdb46bcb94d",
+                                        "user_id": null
+                                    }
+                                },
+                                "result": {
+                                    "event": "another_event",
+                                    "event_data": {}
+                                }
+                            }
+                        ]
+                    },
+                    "config": {
+                        "id": "moon",
+                        "trigger": [
+                            {
+                                "platform": "event",
+                                "event_type": "test_event2"
+                            },
+                            {
+                                "platform": "event",
+                                "event_type": "test_event3"
+                            }
+                        ],
+                        "condition": {
+                            "condition": "template",
+                            "value_template": "{{ trigger.event.event_type=='test_event2' }}"
+                        },
+                        "action": {
+                            "event": "another_event"
+                        }
+                    },
+                    "blueprint_inputs": null,
+                    "context": {
+                        "id": "3128b5fa3494cb17cfb485176ef2cee3",
+                        "parent_id": "66a59f97502785c544724fdb46bcb94d",
+                        "user_id": null
+                    }
+                },
+                "short_dict": {
+                    "last_step": "action/0",
+                    "run_id": "f71d7fa261d361ed999c1dda0a846c99",
+                    "state": "stopped",
+                    "script_execution": "finished",
+                    "timestamp": {
+                        "start": "2021-10-14T06:43:39.551485+00:00",
+                        "finish": "2021-10-14T06:43:39.552822+00:00"
+                    },
+                    "domain": "automation",
+                    "item_id": "moon",
+                    "trigger": "event 'test_event2'"
+                }
+            }
+        ]
+    }
+}

tests/fixtures/trace/script_saved_traces.json

@@ -0,0 +1,165 @@
+{
+    "version": 1,
+    "key": "trace.saved_traces",
+    "data": {
+        "script.sun": [
+            {
+                "extended_dict": {
+                    "last_step": "sequence/0",
+                    "run_id": "6bd24c3b715333fd2192c9501b77664a",
+                    "state": "stopped",
+                    "script_execution": "error",
+                    "timestamp": {
+                        "start": "2021-10-14T06:48:18.037973+00:00",
+                        "finish": "2021-10-14T06:48:18.039367+00:00"
+                    },
+                    "domain": "script",
+                    "item_id": "sun",
+                    "error": "Unable to find service test.automation",
+                    "trace": {
+                        "sequence/0": [
+                            {
+                                "path": "sequence/0",
+                                "timestamp": "2021-10-14T06:48:18.038692+00:00",
+                                "changed_variables": {
+                                    "this": {
+                                        "entity_id": "script.sun",
+                                        "state": "off",
+                                        "attributes": {
+                                            "last_triggered": null,
+                                            "mode": "single",
+                                            "current": 0,
+                                            "friendly_name": "sun"
+                                        },
+                                        "last_changed": "2021-10-14T06:48:18.023069+00:00",
+                                        "last_updated": "2021-10-14T06:48:18.023069+00:00",
+                                        "context": {
+                                            "id": "0c28537a7a55a0c43360fda5c86fb63a",
+                                            "parent_id": null,
+                                            "user_id": null
+                                        }
+                                    },
+                                    "context": {
+                                        "id": "436e5cbeb27415fae813d302e2acb168",
+                                        "parent_id": null,
+                                        "user_id": null
+                                    }
+                                },
+                                "error": "Unable to find service test.automation",
+                                "result": {
+                                    "params": {
+                                        "domain": "test",
+                                        "service": "automation",
+                                        "service_data": {},
+                                        "target": {}
+                                    },
+                                    "running_script": false,
+                                    "limit": 10
+                                }
+                            }
+                        ]
+                    },
+                    "config": {
+                        "sequence": {
+                            "service": "test.automation"
+                        }
+                    },
+                    "blueprint_inputs": null,
+                    "context": {
+                        "id": "436e5cbeb27415fae813d302e2acb168",
+                        "parent_id": null,
+                        "user_id": null
+                    }
+                },
+                "short_dict": {
+                    "last_step": "sequence/0",
+                    "run_id": "6bd24c3b715333fd2192c9501b77664a",
+                    "state": "stopped",
+                    "script_execution": "error",
+                    "timestamp": {
+                        "start": "2021-10-14T06:48:18.037973+00:00",
+                        "finish": "2021-10-14T06:48:18.039367+00:00"
+                    },
+                    "domain": "script",
+                    "item_id": "sun",
+                    "error": "Unable to find service test.automation"
+                }
+            }
+        ],
+        "script.moon": [
+            {
+                "extended_dict": {
+                    "last_step": "sequence/0",
+                    "run_id": "76912f5a7f5e7be2300f92523fd3edf7",
+                    "state": "stopped",
+                    "script_execution": "finished",
+                    "timestamp": {
+                        "start": "2021-10-14T06:48:18.045937+00:00",
+                        "finish": "2021-10-14T06:48:18.047293+00:00"
+                    },
+                    "domain": "script",
+                    "item_id": "moon",
+                    "trace": {
+                        "sequence/0": [
+                            {
+                                "path": "sequence/0",
+                                "timestamp": "2021-10-14T06:48:18.046659+00:00",
+                                "changed_variables": {
+                                    "this": {
+                                        "entity_id": "script.moon",
+                                        "state": "off",
+                                        "attributes": {
+                                            "last_triggered": null,
+                                            "mode": "single",
+                                            "current": 0,
+                                            "friendly_name": "moon"
+                                        },
+                                        "last_changed": "2021-10-14T06:48:18.023671+00:00",
+                                        "last_updated": "2021-10-14T06:48:18.023671+00:00",
+                                        "context": {
+                                            "id": "3dcdb3daa596e44bfd10b407f3078ec0",
+                                            "parent_id": null,
+                                            "user_id": null
+                                        }
+                                    },
+                                    "context": {
+                                        "id": "436e5cbeb27415fae813d302e2acb168",
+                                        "parent_id": null,
+                                        "user_id": null
+                                    }
+                                },
+                                "result": {
+                                    "event": "another_event",
+                                    "event_data": {}
+                                }
+                            }
+                        ]
+                    },
+                    "config": {
+                        "sequence": {
+                            "event": "another_event"
+                        }
+                    },
+                    "blueprint_inputs": null,
+                    "context": {
+                        "id": "436e5cbeb27415fae813d302e2acb168",
+                        "parent_id": null,
+                        "user_id": null
+                    }
+                },
+                "short_dict": {
+                    "last_step": "sequence/0",
+                    "run_id": "76912f5a7f5e7be2300f92523fd3edf7",
+                    "state": "stopped",
+                    "script_execution": "finished",
+                    "timestamp": {
+                        "start": "2021-10-14T06:48:18.045937+00:00",
+                        "finish": "2021-10-14T06:48:18.047293+00:00"
+                    },
+                    "domain": "script",
+                    "item_id": "moon"
+                }
+            }
+        ]
+    }
+}