4004867eda
* Start moving parts of yaml utils to own module Move parts of yaml loader out of the single large file and start to create the structure of the yaml loaders in Ansible [0]. [0]: https://github.com/ansible/ansible/tree/devel/lib/ansible/parsing/yaml * Finish yaml migration, update tests and mocks * Move code around to finish the migration * Update the mocks so that `open` is patched in `homeassistant.util.yaml.loader` instead of `homeassistant.util.yaml`. * Updated mypy ignores * Updated external API of `homeasistant.util.yaml`, see below: Checked what part of the api of `homeassistant.util.yaml` was actually called from outside the tests and added an `__ALL__` that contains only these elements. Updated the tests so that references to internal parts of the API (e.g. the yaml module imported into `homeassistant.util.yaml.loader`) are referenced directly from `homeassistant.util.yaml.loader`. In `tests/test_yaml.py` the import `yaml` refers to `homeassistant.util.yaml` and `yaml_loader` refers to `~.loader`. Future work that remains for the next iteration is to create a custom SafeConstructor and refers to that instead of monkey patching `yaml` with custom loaders. * Update mocks in yaml dumper, check_config
309 lines
11 KiB
Python
309 lines
11 KiB
Python
"""Custom loader."""
|
|
import logging
|
|
import os
|
|
import sys
|
|
import fnmatch
|
|
from collections import OrderedDict
|
|
from typing import Union, List, Dict, Iterator, overload, TypeVar
|
|
|
|
import yaml
|
|
|
|
try:
|
|
import keyring
|
|
except ImportError:
|
|
keyring = None
|
|
|
|
try:
|
|
import credstash
|
|
except ImportError:
|
|
credstash = None
|
|
|
|
from homeassistant.exceptions import HomeAssistantError
|
|
|
|
from .const import _SECRET_NAMESPACE, SECRET_YAML
|
|
from .objects import NodeListClass, NodeStrClass
|
|
|
|
|
|
_LOGGER = logging.getLogger(__name__)
|
|
__SECRET_CACHE = {} # type: Dict[str, JSON_TYPE]
|
|
|
|
JSON_TYPE = Union[List, Dict, str] # pylint: disable=invalid-name
|
|
DICT_T = TypeVar('DICT_T', bound=Dict) # pylint: disable=invalid-name
|
|
|
|
|
|
def clear_secret_cache() -> None:
|
|
"""Clear the secret cache.
|
|
|
|
Async friendly.
|
|
"""
|
|
__SECRET_CACHE.clear()
|
|
|
|
|
|
# pylint: disable=too-many-ancestors
|
|
class SafeLineLoader(yaml.SafeLoader):
|
|
"""Loader class that keeps track of line numbers."""
|
|
|
|
def compose_node(self, parent: yaml.nodes.Node,
|
|
index: int) -> yaml.nodes.Node:
|
|
"""Annotate a node with the first line it was seen."""
|
|
last_line = self.line # type: int
|
|
node = super(SafeLineLoader,
|
|
self).compose_node(parent, index) # type: yaml.nodes.Node
|
|
node.__line__ = last_line + 1 # type: ignore
|
|
return node
|
|
|
|
|
|
def load_yaml(fname: str) -> JSON_TYPE:
|
|
"""Load a YAML file."""
|
|
try:
|
|
with open(fname, encoding='utf-8') as conf_file:
|
|
# If configuration file is empty YAML returns None
|
|
# We convert that to an empty dict
|
|
return yaml.load(conf_file, Loader=SafeLineLoader) or OrderedDict()
|
|
except yaml.YAMLError as exc:
|
|
_LOGGER.error(str(exc))
|
|
raise HomeAssistantError(exc)
|
|
except UnicodeDecodeError as exc:
|
|
_LOGGER.error("Unable to read file %s: %s", fname, exc)
|
|
raise HomeAssistantError(exc)
|
|
|
|
|
|
# pylint: disable=pointless-statement
|
|
@overload
|
|
def _add_reference(obj: Union[list, NodeListClass],
|
|
loader: yaml.SafeLoader,
|
|
node: yaml.nodes.Node) -> NodeListClass: ...
|
|
|
|
|
|
@overload # noqa: F811
|
|
def _add_reference(obj: Union[str, NodeStrClass],
|
|
loader: yaml.SafeLoader,
|
|
node: yaml.nodes.Node) -> NodeStrClass: ...
|
|
|
|
|
|
@overload # noqa: F811
|
|
def _add_reference(obj: DICT_T,
|
|
loader: yaml.SafeLoader,
|
|
node: yaml.nodes.Node) -> DICT_T: ...
|
|
# pylint: enable=pointless-statement
|
|
|
|
|
|
def _add_reference(obj, loader: SafeLineLoader, # type: ignore # noqa: F811
|
|
node: yaml.nodes.Node):
|
|
"""Add file reference information to an object."""
|
|
if isinstance(obj, list):
|
|
obj = NodeListClass(obj)
|
|
if isinstance(obj, str):
|
|
obj = NodeStrClass(obj)
|
|
setattr(obj, '__config_file__', loader.name)
|
|
setattr(obj, '__line__', node.start_mark.line)
|
|
return obj
|
|
|
|
|
|
def _include_yaml(loader: SafeLineLoader,
|
|
node: yaml.nodes.Node) -> JSON_TYPE:
|
|
"""Load another YAML file and embeds it using the !include tag.
|
|
|
|
Example:
|
|
device_tracker: !include device_tracker.yaml
|
|
"""
|
|
fname = os.path.join(os.path.dirname(loader.name), node.value)
|
|
return _add_reference(load_yaml(fname), loader, node)
|
|
|
|
|
|
def _is_file_valid(name: str) -> bool:
|
|
"""Decide if a file is valid."""
|
|
return not name.startswith('.')
|
|
|
|
|
|
def _find_files(directory: str, pattern: str) -> Iterator[str]:
|
|
"""Recursively load files in a directory."""
|
|
for root, dirs, files in os.walk(directory, topdown=True):
|
|
dirs[:] = [d for d in dirs if _is_file_valid(d)]
|
|
for basename in sorted(files):
|
|
if _is_file_valid(basename) and fnmatch.fnmatch(basename, pattern):
|
|
filename = os.path.join(root, basename)
|
|
yield filename
|
|
|
|
|
|
def _include_dir_named_yaml(loader: SafeLineLoader,
|
|
node: yaml.nodes.Node) -> OrderedDict:
|
|
"""Load multiple files from directory as a dictionary."""
|
|
mapping = OrderedDict() # type: OrderedDict
|
|
loc = os.path.join(os.path.dirname(loader.name), node.value)
|
|
for fname in _find_files(loc, '*.yaml'):
|
|
filename = os.path.splitext(os.path.basename(fname))[0]
|
|
if os.path.basename(fname) == SECRET_YAML:
|
|
continue
|
|
mapping[filename] = load_yaml(fname)
|
|
return _add_reference(mapping, loader, node)
|
|
|
|
|
|
def _include_dir_merge_named_yaml(loader: SafeLineLoader,
|
|
node: yaml.nodes.Node) -> OrderedDict:
|
|
"""Load multiple files from directory as a merged dictionary."""
|
|
mapping = OrderedDict() # type: OrderedDict
|
|
loc = os.path.join(os.path.dirname(loader.name), node.value)
|
|
for fname in _find_files(loc, '*.yaml'):
|
|
if os.path.basename(fname) == SECRET_YAML:
|
|
continue
|
|
loaded_yaml = load_yaml(fname)
|
|
if isinstance(loaded_yaml, dict):
|
|
mapping.update(loaded_yaml)
|
|
return _add_reference(mapping, loader, node)
|
|
|
|
|
|
def _include_dir_list_yaml(loader: SafeLineLoader,
|
|
node: yaml.nodes.Node) -> List[JSON_TYPE]:
|
|
"""Load multiple files from directory as a list."""
|
|
loc = os.path.join(os.path.dirname(loader.name), node.value)
|
|
return [load_yaml(f) for f in _find_files(loc, '*.yaml')
|
|
if os.path.basename(f) != SECRET_YAML]
|
|
|
|
|
|
def _include_dir_merge_list_yaml(loader: SafeLineLoader,
|
|
node: yaml.nodes.Node) -> JSON_TYPE:
|
|
"""Load multiple files from directory as a merged list."""
|
|
loc = os.path.join(os.path.dirname(loader.name),
|
|
node.value) # type: str
|
|
merged_list = [] # type: List[JSON_TYPE]
|
|
for fname in _find_files(loc, '*.yaml'):
|
|
if os.path.basename(fname) == SECRET_YAML:
|
|
continue
|
|
loaded_yaml = load_yaml(fname)
|
|
if isinstance(loaded_yaml, list):
|
|
merged_list.extend(loaded_yaml)
|
|
return _add_reference(merged_list, loader, node)
|
|
|
|
|
|
def _ordered_dict(loader: SafeLineLoader,
|
|
node: yaml.nodes.MappingNode) -> OrderedDict:
|
|
"""Load YAML mappings into an ordered dictionary to preserve key order."""
|
|
loader.flatten_mapping(node)
|
|
nodes = loader.construct_pairs(node)
|
|
|
|
seen = {} # type: Dict
|
|
for (key, _), (child_node, _) in zip(nodes, node.value):
|
|
line = child_node.start_mark.line
|
|
|
|
try:
|
|
hash(key)
|
|
except TypeError:
|
|
fname = getattr(loader.stream, 'name', '')
|
|
raise yaml.MarkedYAMLError(
|
|
context="invalid key: \"{}\"".format(key),
|
|
context_mark=yaml.Mark(fname, 0, line, -1, None, None)
|
|
)
|
|
|
|
if key in seen:
|
|
fname = getattr(loader.stream, 'name', '')
|
|
_LOGGER.error(
|
|
'YAML file %s contains duplicate key "%s". '
|
|
'Check lines %d and %d.', fname, key, seen[key], line)
|
|
seen[key] = line
|
|
|
|
return _add_reference(OrderedDict(nodes), loader, node)
|
|
|
|
|
|
def _construct_seq(loader: SafeLineLoader, node: yaml.nodes.Node) -> JSON_TYPE:
|
|
"""Add line number and file name to Load YAML sequence."""
|
|
obj, = loader.construct_yaml_seq(node)
|
|
return _add_reference(obj, loader, node)
|
|
|
|
|
|
def _env_var_yaml(loader: SafeLineLoader,
|
|
node: yaml.nodes.Node) -> str:
|
|
"""Load environment variables and embed it into the configuration YAML."""
|
|
args = node.value.split()
|
|
|
|
# Check for a default value
|
|
if len(args) > 1:
|
|
return os.getenv(args[0], ' '.join(args[1:]))
|
|
if args[0] in os.environ:
|
|
return os.environ[args[0]]
|
|
_LOGGER.error("Environment variable %s not defined.", node.value)
|
|
raise HomeAssistantError(node.value)
|
|
|
|
|
|
def _load_secret_yaml(secret_path: str) -> JSON_TYPE:
|
|
"""Load the secrets yaml from path."""
|
|
secret_path = os.path.join(secret_path, SECRET_YAML)
|
|
if secret_path in __SECRET_CACHE:
|
|
return __SECRET_CACHE[secret_path]
|
|
|
|
_LOGGER.debug('Loading %s', secret_path)
|
|
try:
|
|
secrets = load_yaml(secret_path)
|
|
if not isinstance(secrets, dict):
|
|
raise HomeAssistantError('Secrets is not a dictionary')
|
|
if 'logger' in secrets:
|
|
logger = str(secrets['logger']).lower()
|
|
if logger == 'debug':
|
|
_LOGGER.setLevel(logging.DEBUG)
|
|
else:
|
|
_LOGGER.error("secrets.yaml: 'logger: debug' expected,"
|
|
" but 'logger: %s' found", logger)
|
|
del secrets['logger']
|
|
except FileNotFoundError:
|
|
secrets = {}
|
|
__SECRET_CACHE[secret_path] = secrets
|
|
return secrets
|
|
|
|
|
|
def secret_yaml(loader: SafeLineLoader,
|
|
node: yaml.nodes.Node) -> JSON_TYPE:
|
|
"""Load secrets and embed it into the configuration YAML."""
|
|
secret_path = os.path.dirname(loader.name)
|
|
while True:
|
|
secrets = _load_secret_yaml(secret_path)
|
|
|
|
if node.value in secrets:
|
|
_LOGGER.debug("Secret %s retrieved from secrets.yaml in "
|
|
"folder %s", node.value, secret_path)
|
|
return secrets[node.value]
|
|
|
|
if secret_path == os.path.dirname(sys.path[0]):
|
|
break # sys.path[0] set to config/deps folder by bootstrap
|
|
|
|
secret_path = os.path.dirname(secret_path)
|
|
if not os.path.exists(secret_path) or len(secret_path) < 5:
|
|
break # Somehow we got past the .homeassistant config folder
|
|
|
|
if keyring:
|
|
# do some keyring stuff
|
|
pwd = keyring.get_password(_SECRET_NAMESPACE, node.value)
|
|
if pwd:
|
|
_LOGGER.debug("Secret %s retrieved from keyring", node.value)
|
|
return pwd
|
|
|
|
global credstash # pylint: disable=invalid-name
|
|
|
|
if credstash:
|
|
# pylint: disable=no-member
|
|
try:
|
|
pwd = credstash.getSecret(node.value, table=_SECRET_NAMESPACE)
|
|
if pwd:
|
|
_LOGGER.debug("Secret %s retrieved from credstash", node.value)
|
|
return pwd
|
|
except credstash.ItemNotFound:
|
|
pass
|
|
except Exception: # pylint: disable=broad-except
|
|
# Catch if package installed and no config
|
|
credstash = None
|
|
|
|
raise HomeAssistantError("Secret {} not defined".format(node.value))
|
|
|
|
|
|
yaml.SafeLoader.add_constructor('!include', _include_yaml)
|
|
yaml.SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
|
|
_ordered_dict)
|
|
yaml.SafeLoader.add_constructor(
|
|
yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_seq)
|
|
yaml.SafeLoader.add_constructor('!env_var', _env_var_yaml)
|
|
yaml.SafeLoader.add_constructor('!secret', secret_yaml)
|
|
yaml.SafeLoader.add_constructor('!include_dir_list', _include_dir_list_yaml)
|
|
yaml.SafeLoader.add_constructor('!include_dir_merge_list',
|
|
_include_dir_merge_list_yaml)
|
|
yaml.SafeLoader.add_constructor('!include_dir_named', _include_dir_named_yaml)
|
|
yaml.SafeLoader.add_constructor('!include_dir_merge_named',
|
|
_include_dir_merge_named_yaml)
|