Limit concurrency of async_get_integration to avoid creating extra threads (#43085)
* Limit concurrency of async_get_integration to avoid creating extra threads Since async_get_integration is waiting on the disk most of the time it would end up creating many new threads because the disk could not deliver the data in time. * pylint
This commit is contained in:
parent
518e462e9a
commit
94bf55e29b
6 changed files with 68 additions and 9 deletions
|
@ -1,12 +1,12 @@
|
|||
"""Asyncio backports for Python 3.6 compatibility."""
|
||||
from asyncio import coroutines, ensure_future, get_running_loop
|
||||
"""Asyncio utilities."""
|
||||
from asyncio import Semaphore, coroutines, ensure_future, gather, get_running_loop
|
||||
from asyncio.events import AbstractEventLoop
|
||||
import concurrent.futures
|
||||
import functools
|
||||
import logging
|
||||
import threading
|
||||
from traceback import extract_stack
|
||||
from typing import Any, Callable, Coroutine, TypeVar
|
||||
from typing import Any, Awaitable, Callable, Coroutine, TypeVar
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
@ -121,3 +121,21 @@ def protect_loop(func: Callable) -> Callable:
|
|||
return func(*args, **kwargs)
|
||||
|
||||
return protected_loop_func
|
||||
|
||||
|
||||
async def gather_with_concurrency(
|
||||
limit: int, *tasks: Any, return_exceptions: bool = False
|
||||
) -> Any:
|
||||
"""Wrap asyncio.gather to limit the number of concurrent tasks.
|
||||
|
||||
From: https://stackoverflow.com/a/61478547/9127614
|
||||
"""
|
||||
semaphore = Semaphore(limit)
|
||||
|
||||
async def sem_task(task: Awaitable[Any]) -> Any:
|
||||
async with semaphore:
|
||||
return await task
|
||||
|
||||
return await gather(
|
||||
*(sem_task(task) for task in tasks), return_exceptions=return_exceptions
|
||||
)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue