Support serving of backend translations (#12453)

* Add view to support backend translation fetching

* Load backend translations from component json

* Translations for season sensor

* Scripts to merge and unpack Lokalise translations

* Fix copy paste error

* Serve post-lokalise translations to frontend

* Linting

* Auto-deploy translations with Travis

* Commit post-lokalise translation files

* Split logic into more helper functions

* Fall back to English for missing keys

* Move local translation copies to `.translations`

* Linting

* Initial tests

* Remove unnecessary file check

* Convert translation helper to async/await

* Convert translation helper tests to async/await

* Use set subtraction to find missing_components

* load_translation_files use component->file mapping

* Remove duplicated resources fetching

Get to take advantage of the slick Python 3.5 dict merging here.

* Switch to live project ID

script/translations_download

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Safe bash settings
+# -e            Exit on command fail
+# -u            Exit on unset variable
+# -o pipefail   Exit if piped command has error code
+set -eu -o pipefail
+
+cd "$(dirname "$0")/.."
+
+if [ -z "${LOKALISE_TOKEN-}" ] && [ ! -f .lokalise_token ] ; then
+    echo "Lokalise API token is required to download the latest set of" \
+        "translations. Please create an account by using the following link:" \
+        "https://lokalise.co/signup/130246255a974bd3b5e8a1.51616605/all/" \
+        "Place your token in a new file \".lokalise_token\" in the repo" \
+        "root directory."
+    exit 1
+fi
+
+# Load token from file if not already in the environment
+[ -z "${LOKALISE_TOKEN-}" ] && LOKALISE_TOKEN="$(<.lokalise_token)"
+
+PROJECT_ID="130246255a974bd3b5e8a1.51616605"
+LOCAL_DIR="$(pwd)/build/translations-download"
+FILE_FORMAT=json
+
+mkdir -p ${LOCAL_DIR}
+
+docker pull lokalise/lokalise-cli
+docker run \
+     -v ${LOCAL_DIR}:/opt/dest/locale \
+     lokalise/lokalise-cli lokalise \
+     --token ${LOKALISE_TOKEN} \
+     export ${PROJECT_ID} \
+     --export_empty skip \
+     --type json \
+     --unzip_to /opt/dest
+
+script/translations_download_split.py

script/translations_download_split.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Merge all translation sources into a single JSON file."""
+import glob
+import os
+import re
+
+from homeassistant.util import json as json_util
+
+FILENAME_FORMAT = re.compile(r'strings\.(?P<suffix>\w+)\.json')
+
+
+def get_language(path):
+    """Get the language code for the given file path."""
+    return os.path.splitext(os.path.basename(path))[0]
+
+
+def get_component_path(lang, component):
+    """Get the component translation path."""
+    if os.path.isdir(os.path.join("homeassistant", "components", component)):
+        return os.path.join(
+            "homeassistant", "components", component, ".translations",
+            "{}.json".format(lang))
+    else:
+        return os.path.join(
+            "homeassistant", "components", ".translations",
+            "{}.{}.json".format(component, lang))
+
+
+def get_platform_path(lang, component, platform):
+    """Get the platform translation path."""
+    if os.path.isdir(os.path.join(
+            "homeassistant", "components", component, platform)):
+        return os.path.join(
+            "homeassistant", "components", component, platform,
+            ".translations", "{}.json".format(lang))
+    else:
+        return os.path.join(
+            "homeassistant", "components", component, ".translations",
+            "{}.{}.json".format(platform, lang))
+
+
+def get_component_translations(translations):
+    """Get the component level translations."""
+    translations = translations.copy()
+    translations.pop('platform', None)
+
+    return translations
+
+
+def save_language_translations(lang, translations):
+    """Distribute the translations for this language."""
+    components = translations.get('component', {})
+    for component, component_translations in components.items():
+        base_translations = get_component_translations(component_translations)
+        if base_translations:
+            path = get_component_path(lang, component)
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            json_util.save_json(path, base_translations)
+
+        for platform, platform_translations in component_translations.get(
+                'platform', {}).items():
+            path = get_platform_path(lang, component, platform)
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            json_util.save_json(path, platform_translations)
+
+
+def main():
+    """Main section of the script."""
+    if not os.path.isfile("requirements_all.txt"):
+        print("Run this from HA root dir")
+        return
+
+    paths = glob.iglob("build/translations-download/*.json")
+    for path in paths:
+        lang = get_language(path)
+        translations = json_util.load_json(path)
+        save_language_translations(lang, translations)
+
+
+if __name__ == '__main__':
+    main()

script/translations_upload

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Safe bash settings
+# -e            Exit on command fail
+# -u            Exit on unset variable
+# -o pipefail   Exit if piped command has error code
+set -eu -o pipefail
+
+cd "$(dirname "$0")/.."
+
+if [ -z "${LOKALISE_TOKEN-}" ] && [ ! -f .lokalise_token ] ; then
+    echo "Lokalise API token is required to download the latest set of" \
+        "translations. Please create an account by using the following link:" \
+        "https://lokalise.co/signup/130246255a974bd3b5e8a1.51616605/all/" \
+        "Place your token in a new file \".lokalise_token\" in the repo" \
+        "root directory."
+    exit 1
+fi
+
+# Load token from file if not already in the environment
+[ -z "${LOKALISE_TOKEN-}" ] && LOKALISE_TOKEN="$(<.lokalise_token)"
+
+PROJECT_ID="130246255a974bd3b5e8a1.51616605"
+LOCAL_FILE="$(pwd)/build/translations-upload.json"
+LANG_ISO=en
+
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
+if [ "${CURRENT_BRANCH-}" != "dev" ] && [ "${TRAVIS_BRANCH-}" != "dev" ] ; then
+  echo "Please only run the translations upload script from a clean checkout of dev."
+  exit 1
+fi
+
+script/translations_upload_merge.py
+
+docker pull lokalise/lokalise-cli
+docker run \
+    -v ${LOCAL_FILE}:/opt/src/${LOCAL_FILE} \
+    lokalise/lokalise-cli lokalise \
+    --token ${LOKALISE_TOKEN} \
+    import ${PROJECT_ID} \
+    --file /opt/src/${LOCAL_FILE} \
+    --lang_iso ${LANG_ISO} \
+    --replace 1

script/translations_upload_merge.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Merge all translation sources into a single JSON file."""
+import glob
+import itertools
+import os
+import re
+
+from homeassistant.util import json as json_util
+
+FILENAME_FORMAT = re.compile(r'strings\.(?P<suffix>\w+)\.json')
+
+
+def find_strings_files():
+    """Return the paths of the strings source files."""
+    return itertools.chain(
+        glob.iglob("strings*.json"),
+        glob.iglob("*{}strings*.json".format(os.sep)),
+    )
+
+
+def get_component_platform(path):
+    """Get the component and platform name from the path."""
+    directory, filename = os.path.split(path)
+    match = FILENAME_FORMAT.search(filename)
+    suffix = match.group('suffix') if match else None
+    if directory:
+        return directory, suffix
+    else:
+        return suffix, None
+
+
+def get_translation_dict(translations, component, platform):
+    """Return the dict to hold component translations."""
+    if not component:
+        return translations['component']
+
+    if component not in translations:
+        translations['component'][component] = {}
+
+    if not platform:
+        return translations['component'][component]
+
+    if 'platform' not in translations['component'][component]:
+        translations['component'][component]['platform'] = {}
+
+    if platform not in translations['component'][component]['platform']:
+        translations['component'][component]['platform'][platform] = {}
+
+    return translations['component'][component]['platform'][platform]
+
+
+def main():
+    """Main section of the script."""
+    if not os.path.isfile("requirements_all.txt"):
+        print("Run this from HA root dir")
+        return
+
+    root = os.getcwd()
+    os.chdir(os.path.join("homeassistant", "components"))
+
+    translations = {
+        'component': {}
+    }
+
+    paths = find_strings_files()
+    for path in paths:
+        component, platform = get_component_platform(path)
+        parent = get_translation_dict(translations, component, platform)
+        strings = json_util.load_json(path)
+        parent.update(strings)
+
+    os.chdir(root)
+
+    os.makedirs("build", exist_ok=True)
+
+    json_util.save_json(
+        os.path.join("build", "translations-upload.json"), translations)
+
+
+if __name__ == '__main__':
+    main()

script/travis_deploy

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Safe bash settings
+# -e            Exit on command fail
+# -u            Exit on unset variable
+# -o pipefail   Exit if piped command has error code
+set -eu -o pipefail
+
+cd "$(dirname "$0")/.."
+
+script/translations_upload