Auto repair incorrect collation on MySQL schema (#92270)

* Auto repair incorrect collation on MySQL schema

As we do more union queries in 2023.5.x if there is a mismatch
between collations on tables, they will fail with an error
that is hard for the user to figure out how to fix

`Error executing query: (MySQLdb.OperationalError) (1271, "Illegal mix of collations for operation UNION")`

This was reported in the #beta channel and by PM from others
so the problem is not isolated to a single user

1100908739

* test with ascii since older maraidb versions may not work otherwise

* Revert "test with ascii since older maraidb versions may not work otherwise"

This reverts commit 787fda1aefcd8418a28a8a8f430e7e7232218ef8.t

* older version need to check collation_server because the collation is not reflected if its the default

tests/components/recorder/auto_repairs/events/test_schema.py

@@ -74,3 +74,32 @@ async def test_validate_db_schema_fix_utf8_issue_event_data(
         "Updating character set and collation of table event_data to utf8mb4"
         in caplog.text
     )
+
+
+@pytest.mark.parametrize("enable_schema_validation", [True])
+async def test_validate_db_schema_fix_collation_issue(
+    async_setup_recorder_instance: RecorderInstanceGenerator,
+    hass: HomeAssistant,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test validating DB schema with MySQL.
+
+    Note: The test uses SQLite, the purpose is only to exercise the code.
+    """
+    with patch(
+        "homeassistant.components.recorder.core.Recorder.dialect_name", "mysql"
+    ), patch(
+        "homeassistant.components.recorder.auto_repairs.schema._validate_table_schema_has_correct_collation",
+        return_value={"events.utf8mb4_unicode_ci"},
+    ):
+        await async_setup_recorder_instance(hass)
+        await async_wait_recording_done(hass)
+
+    assert "Schema validation failed" not in caplog.text
+    assert (
+        "Database is about to correct DB schema errors: events.utf8mb4_unicode_ci"
+        in caplog.text
+    )
+    assert (
+        "Updating character set and collation of table events to utf8mb4" in caplog.text
+    )

tests/components/recorder/auto_repairs/states/test_schema.py

@@ -104,3 +104,32 @@ async def test_validate_db_schema_fix_utf8_issue_state_attributes(
         "Updating character set and collation of table state_attributes to utf8mb4"
         in caplog.text
     )
+
+
+@pytest.mark.parametrize("enable_schema_validation", [True])
+async def test_validate_db_schema_fix_collation_issue(
+    async_setup_recorder_instance: RecorderInstanceGenerator,
+    hass: HomeAssistant,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test validating DB schema with MySQL.
+
+    Note: The test uses SQLite, the purpose is only to exercise the code.
+    """
+    with patch(
+        "homeassistant.components.recorder.core.Recorder.dialect_name", "mysql"
+    ), patch(
+        "homeassistant.components.recorder.auto_repairs.schema._validate_table_schema_has_correct_collation",
+        return_value={"states.utf8mb4_unicode_ci"},
+    ):
+        await async_setup_recorder_instance(hass)
+        await async_wait_recording_done(hass)
+
+    assert "Schema validation failed" not in caplog.text
+    assert (
+        "Database is about to correct DB schema errors: states.utf8mb4_unicode_ci"
+        in caplog.text
+    )
+    assert (
+        "Updating character set and collation of table states to utf8mb4" in caplog.text
+    )

tests/components/recorder/auto_repairs/statistics/test_schema.py

@@ -83,3 +83,33 @@ async def test_validate_db_schema_fix_float_issue(
         "sum DOUBLE PRECISION",
     ]
     modify_columns_mock.assert_called_once_with(ANY, ANY, table, modification)
+
+
+@pytest.mark.parametrize("enable_schema_validation", [True])
+async def test_validate_db_schema_fix_collation_issue(
+    async_setup_recorder_instance: RecorderInstanceGenerator,
+    hass: HomeAssistant,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test validating DB schema with MySQL.
+
+    Note: The test uses SQLite, the purpose is only to exercise the code.
+    """
+    with patch(
+        "homeassistant.components.recorder.core.Recorder.dialect_name", "mysql"
+    ), patch(
+        "homeassistant.components.recorder.auto_repairs.schema._validate_table_schema_has_correct_collation",
+        return_value={"statistics.utf8mb4_unicode_ci"},
+    ):
+        await async_setup_recorder_instance(hass)
+        await async_wait_recording_done(hass)
+
+    assert "Schema validation failed" not in caplog.text
+    assert (
+        "Database is about to correct DB schema errors: statistics.utf8mb4_unicode_ci"
+        in caplog.text
+    )
+    assert (
+        "Updating character set and collation of table statistics to utf8mb4"
+        in caplog.text
+    )

tests/components/recorder/auto_repairs/test_schema.py

@@ -10,6 +10,7 @@ from homeassistant.components.recorder.auto_repairs.schema import (
     correct_db_schema_precision,
     correct_db_schema_utf8,
     validate_db_schema_precision,
+    validate_table_schema_has_correct_collation,
     validate_table_schema_supports_utf8,
 )
 from homeassistant.components.recorder.db_schema import States
@@ -106,6 +107,69 @@ async def test_validate_db_schema_fix_utf8_issue_with_broken_schema(
     assert schema_errors == set()
 
 
+async def test_validate_db_schema_fix_incorrect_collation(
+    async_setup_recorder_instance: RecorderInstanceGenerator,
+    hass: HomeAssistant,
+    recorder_db_url: str,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test validating DB schema with MySQL when the collation is incorrect."""
+    if not recorder_db_url.startswith("mysql://"):
+        # This problem only happens on MySQL
+        return
+    await async_setup_recorder_instance(hass)
+    await async_wait_recording_done(hass)
+    instance = get_instance(hass)
+    session_maker = instance.get_session
+
+    def _break_states_schema():
+        with session_scope(session=session_maker()) as session:
+            session.execute(
+                text(
+                    "ALTER TABLE states CHARACTER SET utf8mb3 COLLATE utf8_general_ci, "
+                    "LOCK=EXCLUSIVE;"
+                )
+            )
+
+    await instance.async_add_executor_job(_break_states_schema)
+    schema_errors = await instance.async_add_executor_job(
+        validate_table_schema_has_correct_collation, instance, States
+    )
+    assert schema_errors == {"states.utf8mb4_unicode_ci"}
+
+    # Now repair the schema
+    await instance.async_add_executor_job(
+        correct_db_schema_utf8, instance, States, schema_errors
+    )
+
+    # Now validate the schema again
+    schema_errors = await instance.async_add_executor_job(
+        validate_table_schema_has_correct_collation, instance, States
+    )
+    assert schema_errors == set()
+
+
+async def test_validate_db_schema_precision_correct_collation(
+    async_setup_recorder_instance: RecorderInstanceGenerator,
+    hass: HomeAssistant,
+    recorder_db_url: str,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Test validating DB schema when the schema is correct with the correct collation."""
+    if not recorder_db_url.startswith("mysql://"):
+        # This problem only happens on MySQL
+        return
+    await async_setup_recorder_instance(hass)
+    await async_wait_recording_done(hass)
+    instance = get_instance(hass)
+    schema_errors = await instance.async_add_executor_job(
+        validate_table_schema_has_correct_collation,
+        instance,
+        States,
+    )
+    assert schema_errors == set()
+
+
 async def test_validate_db_schema_fix_utf8_issue_with_broken_schema_unrepairable(
     async_setup_recorder_instance: RecorderInstanceGenerator,
     hass: HomeAssistant,