2016-03-07 23:20:48 +01:00
|
|
|
"""Helper methods for various modules."""
|
2021-03-17 21:46:07 +01:00
|
|
|
from __future__ import annotations
|
|
|
|
|
2018-03-09 19:38:51 -08:00
|
|
|
import asyncio
|
2022-02-04 14:45:25 -08:00
|
|
|
from collections.abc import Callable, Coroutine, Iterable, KeysView, Mapping
|
2018-07-23 11:24:39 +03:00
|
|
|
from datetime import datetime, timedelta
|
2021-11-15 18:18:57 +01:00
|
|
|
from functools import wraps
|
2015-01-17 21:55:33 -08:00
|
|
|
import random
|
2019-12-09 16:42:10 +01:00
|
|
|
import re
|
2015-01-17 21:55:33 -08:00
|
|
|
import string
|
2019-12-09 16:42:10 +01:00
|
|
|
import threading
|
2021-09-29 16:32:11 +02:00
|
|
|
from typing import Any, TypeVar
|
2016-07-23 13:07:08 -05:00
|
|
|
|
2018-12-17 07:51:13 +01:00
|
|
|
import slugify as unicode_slug
|
|
|
|
|
2016-04-16 00:55:35 -07:00
|
|
|
from .dt import as_local, utcnow
|
2015-04-28 19:12:05 -07:00
|
|
|
|
2022-03-17 18:52:38 +01:00
|
|
|
_T = TypeVar("_T")
|
|
|
|
_U = TypeVar("_U")
|
2016-08-07 18:26:35 -05:00
|
|
|
|
2019-07-30 16:59:12 -07:00
|
|
|
RE_SANITIZE_FILENAME = re.compile(r"(~|\.\.|/|\\)")
|
|
|
|
RE_SANITIZE_PATH = re.compile(r"(~|\.(\.)+)")
|
2013-12-07 12:54:19 -08:00
|
|
|
|
2013-11-10 16:46:48 -08:00
|
|
|
|
2021-01-26 15:53:21 +01:00
|
|
|
def raise_if_invalid_filename(filename: str) -> None:
|
2023-01-09 07:01:55 +01:00
|
|
|
"""Check if a filename is valid.
|
2021-01-26 15:53:21 +01:00
|
|
|
|
|
|
|
Raises a ValueError if the filename is invalid.
|
|
|
|
"""
|
|
|
|
if RE_SANITIZE_FILENAME.sub("", filename) != filename:
|
|
|
|
raise ValueError(f"{filename} is not a safe filename")
|
|
|
|
|
|
|
|
|
|
|
|
def raise_if_invalid_path(path: str) -> None:
|
2023-01-09 07:01:55 +01:00
|
|
|
"""Check if a path is valid.
|
2021-01-26 15:53:21 +01:00
|
|
|
|
|
|
|
Raises a ValueError if the path is invalid.
|
|
|
|
"""
|
|
|
|
if RE_SANITIZE_PATH.sub("", path) != path:
|
|
|
|
raise ValueError(f"{path} is not a safe path")
|
|
|
|
|
|
|
|
|
2021-07-28 11:50:13 +02:00
|
|
|
def slugify(text: str | None, *, separator: str = "_") -> str:
|
2016-03-07 23:20:48 +01:00
|
|
|
"""Slugify a given text."""
|
2021-07-28 11:50:13 +02:00
|
|
|
if text == "" or text is None:
|
2021-01-27 12:25:49 +01:00
|
|
|
return ""
|
|
|
|
slug = unicode_slug.slugify(text, separator=separator)
|
|
|
|
return "unknown" if slug == "" else slug
|
2014-01-19 19:10:40 -08:00
|
|
|
|
|
|
|
|
2016-07-23 13:07:08 -05:00
|
|
|
def repr_helper(inp: Any) -> str:
|
2016-03-07 23:20:48 +01:00
|
|
|
"""Help creating a more readable string representation of objects."""
|
2022-02-04 14:45:25 -08:00
|
|
|
if isinstance(inp, Mapping):
|
2014-04-14 00:10:24 -07:00
|
|
|
return ", ".join(
|
2020-04-07 23:14:28 +02:00
|
|
|
f"{repr_helper(key)}={repr_helper(item)}" for key, item in inp.items()
|
2019-07-30 16:59:12 -07:00
|
|
|
)
|
2018-07-23 11:16:05 +03:00
|
|
|
if isinstance(inp, datetime):
|
2016-04-16 00:55:35 -07:00
|
|
|
return as_local(inp).isoformat()
|
2017-07-05 20:02:16 -07:00
|
|
|
|
|
|
|
return str(inp)
|
2014-01-26 18:44:36 -08:00
|
|
|
|
|
|
|
|
2019-07-30 16:59:12 -07:00
|
|
|
def convert(
|
2022-03-17 18:52:38 +01:00
|
|
|
value: _T | None, to_type: Callable[[_T], _U], default: _U | None = None
|
|
|
|
) -> _U | None:
|
2016-03-07 23:20:48 +01:00
|
|
|
"""Convert value to to_type, returns default if fails."""
|
2014-03-16 15:00:59 -07:00
|
|
|
try:
|
2014-03-26 00:08:50 -07:00
|
|
|
return default if value is None else to_type(value)
|
2016-02-21 11:23:16 -08:00
|
|
|
except (ValueError, TypeError):
|
2014-03-16 15:00:59 -07:00
|
|
|
# If value could not be converted
|
|
|
|
return default
|
|
|
|
|
|
|
|
|
2019-07-30 16:59:12 -07:00
|
|
|
def ensure_unique_string(
|
2021-03-17 21:46:07 +01:00
|
|
|
preferred_string: str, current_strings: Iterable[str] | KeysView[str]
|
2019-07-30 16:59:12 -07:00
|
|
|
) -> str:
|
2016-03-07 23:20:48 +01:00
|
|
|
"""Return a string that is not present in current_strings.
|
|
|
|
|
|
|
|
If preferred string exists will append _2, _3, ..
|
|
|
|
"""
|
2015-01-17 21:55:33 -08:00
|
|
|
test_string = preferred_string
|
2016-07-27 22:33:49 -05:00
|
|
|
current_strings_set = set(current_strings)
|
2014-03-23 12:31:24 -07:00
|
|
|
|
|
|
|
tries = 1
|
|
|
|
|
2016-07-27 22:33:49 -05:00
|
|
|
while test_string in current_strings_set:
|
2014-03-23 12:31:24 -07:00
|
|
|
tries += 1
|
2019-08-23 18:53:33 +02:00
|
|
|
test_string = f"{preferred_string}_{tries}"
|
2014-03-23 12:31:24 -07:00
|
|
|
|
2015-01-17 21:55:33 -08:00
|
|
|
return test_string
|
2014-03-23 12:31:24 -07:00
|
|
|
|
|
|
|
|
2015-01-17 21:55:33 -08:00
|
|
|
# Taken from http://stackoverflow.com/a/23728630
|
2018-07-23 11:24:39 +03:00
|
|
|
def get_random_string(length: int = 10) -> str:
|
2016-03-07 23:20:48 +01:00
|
|
|
"""Return a random string with letters and digits."""
|
2015-01-17 21:55:33 -08:00
|
|
|
generator = random.SystemRandom()
|
|
|
|
source_chars = string.ascii_letters + string.digits
|
|
|
|
|
2019-07-30 16:59:12 -07:00
|
|
|
return "".join(generator.choice(source_chars) for _ in range(length))
|
2015-01-17 21:55:33 -08:00
|
|
|
|
|
|
|
|
2018-07-20 11:45:20 +03:00
|
|
|
class Throttle:
|
2016-03-07 23:20:48 +01:00
|
|
|
"""A class for throttling the execution of tasks.
|
|
|
|
|
|
|
|
This method decorator adds a cooldown to a method to prevent it from being
|
2022-12-20 11:10:31 +01:00
|
|
|
called more than 1 time within the timedelta interval `min_time` after it
|
2014-12-04 21:06:45 -08:00
|
|
|
returned its result.
|
2014-12-04 01:14:27 -08:00
|
|
|
|
2014-12-04 21:06:45 -08:00
|
|
|
Calling a method a second time during the interval will return None.
|
2014-12-04 01:14:27 -08:00
|
|
|
|
2014-12-04 21:06:45 -08:00
|
|
|
Pass keyword argument `no_throttle=True` to the wrapped method to make
|
|
|
|
the call not throttled.
|
|
|
|
|
|
|
|
Decorator takes in an optional second timedelta interval to throttle the
|
|
|
|
'no_throttle' calls.
|
|
|
|
|
|
|
|
Adds a datetime attribute `last_call` to the method.
|
2014-12-04 01:14:27 -08:00
|
|
|
"""
|
|
|
|
|
2019-07-30 16:59:12 -07:00
|
|
|
def __init__(
|
2021-03-17 21:46:07 +01:00
|
|
|
self, min_time: timedelta, limit_no_throttle: timedelta | None = None
|
2019-07-30 16:59:12 -07:00
|
|
|
) -> None:
|
2016-03-07 23:20:48 +01:00
|
|
|
"""Initialize the throttle."""
|
2014-12-04 01:14:27 -08:00
|
|
|
self.min_time = min_time
|
2014-12-04 21:06:45 -08:00
|
|
|
self.limit_no_throttle = limit_no_throttle
|
2014-12-04 01:14:27 -08:00
|
|
|
|
2018-07-23 11:24:39 +03:00
|
|
|
def __call__(self, method: Callable) -> Callable:
|
2016-03-07 23:20:48 +01:00
|
|
|
"""Caller for the throttle."""
|
2018-03-16 20:27:05 -07:00
|
|
|
# Make sure we return a coroutine if the method is async.
|
|
|
|
if asyncio.iscoroutinefunction(method):
|
2019-07-30 16:59:12 -07:00
|
|
|
|
2018-07-23 11:24:39 +03:00
|
|
|
async def throttled_value() -> None:
|
2018-03-16 20:27:05 -07:00
|
|
|
"""Stand-in function for when real func is being throttled."""
|
|
|
|
return None
|
2019-07-30 16:59:12 -07:00
|
|
|
|
2018-03-16 20:27:05 -07:00
|
|
|
else:
|
2019-07-30 16:59:12 -07:00
|
|
|
|
2022-02-18 11:30:59 +01:00
|
|
|
def throttled_value() -> None: # type: ignore[misc]
|
2018-03-16 20:27:05 -07:00
|
|
|
"""Stand-in function for when real func is being throttled."""
|
|
|
|
return None
|
|
|
|
|
2014-12-04 21:06:45 -08:00
|
|
|
if self.limit_no_throttle is not None:
|
|
|
|
method = Throttle(self.limit_no_throttle)(method)
|
|
|
|
|
2015-10-11 10:42:42 -07:00
|
|
|
# Different methods that can be passed in:
|
|
|
|
# - a function
|
|
|
|
# - an unbound function on a class
|
|
|
|
# - a method (bound function on a class)
|
|
|
|
|
|
|
|
# We want to be able to differentiate between function and unbound
|
|
|
|
# methods (which are considered functions).
|
2017-09-23 17:15:46 +02:00
|
|
|
# All methods have the classname in their qualname separated by a '.'
|
2015-10-08 23:49:55 -07:00
|
|
|
# Functions have a '.' in their qualname if defined inline, but will
|
|
|
|
# be prefixed by '.<locals>.' so we strip that out.
|
2019-07-30 16:59:12 -07:00
|
|
|
is_func = (
|
|
|
|
not hasattr(method, "__self__")
|
2022-11-15 22:45:48 +02:00
|
|
|
and "." not in method.__qualname__.rpartition(".<locals>.")[-1]
|
2019-07-30 16:59:12 -07:00
|
|
|
)
|
2015-10-08 23:49:55 -07:00
|
|
|
|
2014-12-04 01:14:27 -08:00
|
|
|
@wraps(method)
|
2021-03-17 21:46:07 +01:00
|
|
|
def wrapper(*args: Any, **kwargs: Any) -> Callable | Coroutine:
|
2017-05-02 18:18:47 +02:00
|
|
|
"""Wrap that allows wrapped to be called only once per min_time.
|
2016-03-07 23:20:48 +01:00
|
|
|
|
2015-01-05 20:50:34 -08:00
|
|
|
If we cannot acquire the lock, it is running so return None.
|
2014-12-04 01:14:27 -08:00
|
|
|
"""
|
2019-07-30 16:59:12 -07:00
|
|
|
if hasattr(method, "__self__"):
|
|
|
|
host = getattr(method, "__self__")
|
2015-10-11 10:42:42 -07:00
|
|
|
elif is_func:
|
|
|
|
host = wrapper
|
|
|
|
else:
|
|
|
|
host = args[0] if args else wrapper
|
|
|
|
|
2023-01-20 13:47:55 +01:00
|
|
|
# pylint: disable=protected-access
|
2019-07-30 16:59:12 -07:00
|
|
|
if not hasattr(host, "_throttle"):
|
2016-02-27 23:18:56 +01:00
|
|
|
host._throttle = {}
|
2015-10-08 23:49:55 -07:00
|
|
|
|
2016-02-27 23:18:56 +01:00
|
|
|
if id(self) not in host._throttle:
|
|
|
|
host._throttle[id(self)] = [threading.Lock(), None]
|
|
|
|
throttle = host._throttle[id(self)]
|
2020-05-09 14:08:40 +03:00
|
|
|
# pylint: enable=protected-access
|
2016-02-27 23:18:56 +01:00
|
|
|
|
|
|
|
if not throttle[0].acquire(False):
|
2018-03-09 19:38:51 -08:00
|
|
|
return throttled_value()
|
2015-09-12 22:56:49 -07:00
|
|
|
|
2015-10-08 23:49:55 -07:00
|
|
|
# Check if method is never called or no_throttle is given
|
2019-07-30 16:59:12 -07:00
|
|
|
force = kwargs.pop("no_throttle", False) or not throttle[1]
|
2015-09-12 22:56:49 -07:00
|
|
|
|
2015-10-08 23:49:55 -07:00
|
|
|
try:
|
2016-02-27 23:18:56 +01:00
|
|
|
if force or utcnow() - throttle[1] > self.min_time:
|
2015-09-12 22:56:49 -07:00
|
|
|
result = method(*args, **kwargs)
|
2016-02-27 23:18:56 +01:00
|
|
|
throttle[1] = utcnow()
|
2022-02-18 11:30:59 +01:00
|
|
|
return result # type: ignore[no-any-return]
|
2017-07-05 20:02:16 -07:00
|
|
|
|
2018-03-09 19:38:51 -08:00
|
|
|
return throttled_value()
|
2015-09-12 22:56:49 -07:00
|
|
|
finally:
|
2016-02-27 23:18:56 +01:00
|
|
|
throttle[0].release()
|
2014-12-04 01:14:27 -08:00
|
|
|
|
|
|
|
return wrapper
|