helpers.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986
  1. """Various helper functions"""
  2. import asyncio
  3. import base64
  4. import binascii
  5. import contextlib
  6. import datetime
  7. import enum
  8. import functools
  9. import inspect
  10. import netrc
  11. import os
  12. import platform
  13. import re
  14. import sys
  15. import time
  16. import weakref
  17. from collections import namedtuple
  18. from contextlib import suppress
  19. from email.message import EmailMessage
  20. from email.parser import HeaderParser
  21. from email.policy import HTTP
  22. from email.utils import parsedate
  23. from math import ceil
  24. from pathlib import Path
  25. from types import MappingProxyType, TracebackType
  26. from typing import (
  27. Any,
  28. Callable,
  29. ContextManager,
  30. Dict,
  31. Generator,
  32. Generic,
  33. Iterable,
  34. Iterator,
  35. List,
  36. Mapping,
  37. Optional,
  38. Protocol,
  39. Tuple,
  40. Type,
  41. TypeVar,
  42. Union,
  43. get_args,
  44. overload,
  45. )
  46. from urllib.parse import quote
  47. from urllib.request import getproxies, proxy_bypass
  48. import attr
  49. from multidict import MultiDict, MultiDictProxy, MultiMapping
  50. from propcache.api import under_cached_property as reify
  51. from yarl import URL
  52. from . import hdrs
  53. from .log import client_logger
  54. if sys.version_info >= (3, 11):
  55. import asyncio as async_timeout
  56. else:
  57. import async_timeout
  58. __all__ = ("BasicAuth", "ChainMapProxy", "ETag", "reify")
  59. IS_MACOS = platform.system() == "Darwin"
  60. IS_WINDOWS = platform.system() == "Windows"
  61. PY_310 = sys.version_info >= (3, 10)
  62. PY_311 = sys.version_info >= (3, 11)
  63. _T = TypeVar("_T")
  64. _S = TypeVar("_S")
  65. _SENTINEL = enum.Enum("_SENTINEL", "sentinel")
  66. sentinel = _SENTINEL.sentinel
  67. NO_EXTENSIONS = bool(os.environ.get("AIOHTTP_NO_EXTENSIONS"))
  68. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1
  69. EMPTY_BODY_STATUS_CODES = frozenset((204, 304, *range(100, 200)))
  70. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1
  71. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.2
  72. EMPTY_BODY_METHODS = hdrs.METH_HEAD_ALL
  73. DEBUG = sys.flags.dev_mode or (
  74. not sys.flags.ignore_environment and bool(os.environ.get("PYTHONASYNCIODEBUG"))
  75. )
  76. CHAR = {chr(i) for i in range(0, 128)}
  77. CTL = {chr(i) for i in range(0, 32)} | {
  78. chr(127),
  79. }
  80. SEPARATORS = {
  81. "(",
  82. ")",
  83. "<",
  84. ">",
  85. "@",
  86. ",",
  87. ";",
  88. ":",
  89. "\\",
  90. '"',
  91. "/",
  92. "[",
  93. "]",
  94. "?",
  95. "=",
  96. "{",
  97. "}",
  98. " ",
  99. chr(9),
  100. }
  101. TOKEN = CHAR ^ CTL ^ SEPARATORS
  102. class noop:
  103. def __await__(self) -> Generator[None, None, None]:
  104. yield
  105. class BasicAuth(namedtuple("BasicAuth", ["login", "password", "encoding"])):
  106. """Http basic authentication helper."""
  107. def __new__(
  108. cls, login: str, password: str = "", encoding: str = "latin1"
  109. ) -> "BasicAuth":
  110. if login is None:
  111. raise ValueError("None is not allowed as login value")
  112. if password is None:
  113. raise ValueError("None is not allowed as password value")
  114. if ":" in login:
  115. raise ValueError('A ":" is not allowed in login (RFC 1945#section-11.1)')
  116. return super().__new__(cls, login, password, encoding)
  117. @classmethod
  118. def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth":
  119. """Create a BasicAuth object from an Authorization HTTP header."""
  120. try:
  121. auth_type, encoded_credentials = auth_header.split(" ", 1)
  122. except ValueError:
  123. raise ValueError("Could not parse authorization header.")
  124. if auth_type.lower() != "basic":
  125. raise ValueError("Unknown authorization method %s" % auth_type)
  126. try:
  127. decoded = base64.b64decode(
  128. encoded_credentials.encode("ascii"), validate=True
  129. ).decode(encoding)
  130. except binascii.Error:
  131. raise ValueError("Invalid base64 encoding.")
  132. try:
  133. # RFC 2617 HTTP Authentication
  134. # https://www.ietf.org/rfc/rfc2617.txt
  135. # the colon must be present, but the username and password may be
  136. # otherwise blank.
  137. username, password = decoded.split(":", 1)
  138. except ValueError:
  139. raise ValueError("Invalid credentials.")
  140. return cls(username, password, encoding=encoding)
  141. @classmethod
  142. def from_url(cls, url: URL, *, encoding: str = "latin1") -> Optional["BasicAuth"]:
  143. """Create BasicAuth from url."""
  144. if not isinstance(url, URL):
  145. raise TypeError("url should be yarl.URL instance")
  146. # Check raw_user and raw_password first as yarl is likely
  147. # to already have these values parsed from the netloc in the cache.
  148. if url.raw_user is None and url.raw_password is None:
  149. return None
  150. return cls(url.user or "", url.password or "", encoding=encoding)
  151. def encode(self) -> str:
  152. """Encode credentials."""
  153. creds = (f"{self.login}:{self.password}").encode(self.encoding)
  154. return "Basic %s" % base64.b64encode(creds).decode(self.encoding)
  155. def strip_auth_from_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]:
  156. """Remove user and password from URL if present and return BasicAuth object."""
  157. # Check raw_user and raw_password first as yarl is likely
  158. # to already have these values parsed from the netloc in the cache.
  159. if url.raw_user is None and url.raw_password is None:
  160. return url, None
  161. return url.with_user(None), BasicAuth(url.user or "", url.password or "")
  162. def netrc_from_env() -> Optional[netrc.netrc]:
  163. """Load netrc from file.
  164. Attempt to load it from the path specified by the env-var
  165. NETRC or in the default location in the user's home directory.
  166. Returns None if it couldn't be found or fails to parse.
  167. """
  168. netrc_env = os.environ.get("NETRC")
  169. if netrc_env is not None:
  170. netrc_path = Path(netrc_env)
  171. else:
  172. try:
  173. home_dir = Path.home()
  174. except RuntimeError as e: # pragma: no cover
  175. # if pathlib can't resolve home, it may raise a RuntimeError
  176. client_logger.debug(
  177. "Could not resolve home directory when "
  178. "trying to look for .netrc file: %s",
  179. e,
  180. )
  181. return None
  182. netrc_path = home_dir / ("_netrc" if IS_WINDOWS else ".netrc")
  183. try:
  184. return netrc.netrc(str(netrc_path))
  185. except netrc.NetrcParseError as e:
  186. client_logger.warning("Could not parse .netrc file: %s", e)
  187. except OSError as e:
  188. netrc_exists = False
  189. with contextlib.suppress(OSError):
  190. netrc_exists = netrc_path.is_file()
  191. # we couldn't read the file (doesn't exist, permissions, etc.)
  192. if netrc_env or netrc_exists:
  193. # only warn if the environment wanted us to load it,
  194. # or it appears like the default file does actually exist
  195. client_logger.warning("Could not read .netrc file: %s", e)
  196. return None
  197. @attr.s(auto_attribs=True, frozen=True, slots=True)
  198. class ProxyInfo:
  199. proxy: URL
  200. proxy_auth: Optional[BasicAuth]
  201. def basicauth_from_netrc(netrc_obj: Optional[netrc.netrc], host: str) -> BasicAuth:
  202. """
  203. Return :py:class:`~aiohttp.BasicAuth` credentials for ``host`` from ``netrc_obj``.
  204. :raises LookupError: if ``netrc_obj`` is :py:data:`None` or if no
  205. entry is found for the ``host``.
  206. """
  207. if netrc_obj is None:
  208. raise LookupError("No .netrc file found")
  209. auth_from_netrc = netrc_obj.authenticators(host)
  210. if auth_from_netrc is None:
  211. raise LookupError(f"No entry for {host!s} found in the `.netrc` file.")
  212. login, account, password = auth_from_netrc
  213. # TODO(PY311): username = login or account
  214. # Up to python 3.10, account could be None if not specified,
  215. # and login will be empty string if not specified. From 3.11,
  216. # login and account will be empty string if not specified.
  217. username = login if (login or account is None) else account
  218. # TODO(PY311): Remove this, as password will be empty string
  219. # if not specified
  220. if password is None:
  221. password = ""
  222. return BasicAuth(username, password)
  223. def proxies_from_env() -> Dict[str, ProxyInfo]:
  224. proxy_urls = {
  225. k: URL(v)
  226. for k, v in getproxies().items()
  227. if k in ("http", "https", "ws", "wss")
  228. }
  229. netrc_obj = netrc_from_env()
  230. stripped = {k: strip_auth_from_url(v) for k, v in proxy_urls.items()}
  231. ret = {}
  232. for proto, val in stripped.items():
  233. proxy, auth = val
  234. if proxy.scheme in ("https", "wss"):
  235. client_logger.warning(
  236. "%s proxies %s are not supported, ignoring", proxy.scheme.upper(), proxy
  237. )
  238. continue
  239. if netrc_obj and auth is None:
  240. if proxy.host is not None:
  241. try:
  242. auth = basicauth_from_netrc(netrc_obj, proxy.host)
  243. except LookupError:
  244. auth = None
  245. ret[proto] = ProxyInfo(proxy, auth)
  246. return ret
  247. def get_env_proxy_for_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]:
  248. """Get a permitted proxy for the given URL from the env."""
  249. if url.host is not None and proxy_bypass(url.host):
  250. raise LookupError(f"Proxying is disallowed for `{url.host!r}`")
  251. proxies_in_env = proxies_from_env()
  252. try:
  253. proxy_info = proxies_in_env[url.scheme]
  254. except KeyError:
  255. raise LookupError(f"No proxies found for `{url!s}` in the env")
  256. else:
  257. return proxy_info.proxy, proxy_info.proxy_auth
  258. @attr.s(auto_attribs=True, frozen=True, slots=True)
  259. class MimeType:
  260. type: str
  261. subtype: str
  262. suffix: str
  263. parameters: "MultiDictProxy[str]"
  264. @functools.lru_cache(maxsize=56)
  265. def parse_mimetype(mimetype: str) -> MimeType:
  266. """Parses a MIME type into its components.
  267. mimetype is a MIME type string.
  268. Returns a MimeType object.
  269. Example:
  270. >>> parse_mimetype('text/html; charset=utf-8')
  271. MimeType(type='text', subtype='html', suffix='',
  272. parameters={'charset': 'utf-8'})
  273. """
  274. if not mimetype:
  275. return MimeType(
  276. type="", subtype="", suffix="", parameters=MultiDictProxy(MultiDict())
  277. )
  278. parts = mimetype.split(";")
  279. params: MultiDict[str] = MultiDict()
  280. for item in parts[1:]:
  281. if not item:
  282. continue
  283. key, _, value = item.partition("=")
  284. params.add(key.lower().strip(), value.strip(' "'))
  285. fulltype = parts[0].strip().lower()
  286. if fulltype == "*":
  287. fulltype = "*/*"
  288. mtype, _, stype = fulltype.partition("/")
  289. stype, _, suffix = stype.partition("+")
  290. return MimeType(
  291. type=mtype, subtype=stype, suffix=suffix, parameters=MultiDictProxy(params)
  292. )
  293. class EnsureOctetStream(EmailMessage):
  294. def __init__(self) -> None:
  295. super().__init__()
  296. # https://www.rfc-editor.org/rfc/rfc9110#section-8.3-5
  297. self.set_default_type("application/octet-stream")
  298. def get_content_type(self) -> str:
  299. """Re-implementation from Message
  300. Returns application/octet-stream in place of plain/text when
  301. value is wrong.
  302. The way this class is used guarantees that content-type will
  303. be present so simplify the checks wrt to the base implementation.
  304. """
  305. value = self.get("content-type", "").lower()
  306. # Based on the implementation of _splitparam in the standard library
  307. ctype, _, _ = value.partition(";")
  308. ctype = ctype.strip()
  309. if ctype.count("/") != 1:
  310. return self.get_default_type()
  311. return ctype
  312. @functools.lru_cache(maxsize=56)
  313. def parse_content_type(raw: str) -> Tuple[str, MappingProxyType[str, str]]:
  314. """Parse Content-Type header.
  315. Returns a tuple of the parsed content type and a
  316. MappingProxyType of parameters. The default returned value
  317. is `application/octet-stream`
  318. """
  319. msg = HeaderParser(EnsureOctetStream, policy=HTTP).parsestr(f"Content-Type: {raw}")
  320. content_type = msg.get_content_type()
  321. params = msg.get_params(())
  322. content_dict = dict(params[1:]) # First element is content type again
  323. return content_type, MappingProxyType(content_dict)
  324. def guess_filename(obj: Any, default: Optional[str] = None) -> Optional[str]:
  325. name = getattr(obj, "name", None)
  326. if name and isinstance(name, str) and name[0] != "<" and name[-1] != ">":
  327. return Path(name).name
  328. return default
  329. not_qtext_re = re.compile(r"[^\041\043-\133\135-\176]")
  330. QCONTENT = {chr(i) for i in range(0x20, 0x7F)} | {"\t"}
  331. def quoted_string(content: str) -> str:
  332. """Return 7-bit content as quoted-string.
  333. Format content into a quoted-string as defined in RFC5322 for
  334. Internet Message Format. Notice that this is not the 8-bit HTTP
  335. format, but the 7-bit email format. Content must be in usascii or
  336. a ValueError is raised.
  337. """
  338. if not (QCONTENT > set(content)):
  339. raise ValueError(f"bad content for quoted-string {content!r}")
  340. return not_qtext_re.sub(lambda x: "\\" + x.group(0), content)
  341. def content_disposition_header(
  342. disptype: str, quote_fields: bool = True, _charset: str = "utf-8", **params: str
  343. ) -> str:
  344. """Sets ``Content-Disposition`` header for MIME.
  345. This is the MIME payload Content-Disposition header from RFC 2183
  346. and RFC 7579 section 4.2, not the HTTP Content-Disposition from
  347. RFC 6266.
  348. disptype is a disposition type: inline, attachment, form-data.
  349. Should be valid extension token (see RFC 2183)
  350. quote_fields performs value quoting to 7-bit MIME headers
  351. according to RFC 7578. Set to quote_fields to False if recipient
  352. can take 8-bit file names and field values.
  353. _charset specifies the charset to use when quote_fields is True.
  354. params is a dict with disposition params.
  355. """
  356. if not disptype or not (TOKEN > set(disptype)):
  357. raise ValueError(f"bad content disposition type {disptype!r}")
  358. value = disptype
  359. if params:
  360. lparams = []
  361. for key, val in params.items():
  362. if not key or not (TOKEN > set(key)):
  363. raise ValueError(f"bad content disposition parameter {key!r}={val!r}")
  364. if quote_fields:
  365. if key.lower() == "filename":
  366. qval = quote(val, "", encoding=_charset)
  367. lparams.append((key, '"%s"' % qval))
  368. else:
  369. try:
  370. qval = quoted_string(val)
  371. except ValueError:
  372. qval = "".join(
  373. (_charset, "''", quote(val, "", encoding=_charset))
  374. )
  375. lparams.append((key + "*", qval))
  376. else:
  377. lparams.append((key, '"%s"' % qval))
  378. else:
  379. qval = val.replace("\\", "\\\\").replace('"', '\\"')
  380. lparams.append((key, '"%s"' % qval))
  381. sparams = "; ".join("=".join(pair) for pair in lparams)
  382. value = "; ".join((value, sparams))
  383. return value
  384. def is_ip_address(host: Optional[str]) -> bool:
  385. """Check if host looks like an IP Address.
  386. This check is only meant as a heuristic to ensure that
  387. a host is not a domain name.
  388. """
  389. if not host:
  390. return False
  391. # For a host to be an ipv4 address, it must be all numeric.
  392. # The host must contain a colon to be an IPv6 address.
  393. return ":" in host or host.replace(".", "").isdigit()
  394. _cached_current_datetime: Optional[int] = None
  395. _cached_formatted_datetime = ""
  396. def rfc822_formatted_time() -> str:
  397. global _cached_current_datetime
  398. global _cached_formatted_datetime
  399. now = int(time.time())
  400. if now != _cached_current_datetime:
  401. # Weekday and month names for HTTP date/time formatting;
  402. # always English!
  403. # Tuples are constants stored in codeobject!
  404. _weekdayname = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
  405. _monthname = (
  406. "", # Dummy so we can use 1-based month numbers
  407. "Jan",
  408. "Feb",
  409. "Mar",
  410. "Apr",
  411. "May",
  412. "Jun",
  413. "Jul",
  414. "Aug",
  415. "Sep",
  416. "Oct",
  417. "Nov",
  418. "Dec",
  419. )
  420. year, month, day, hh, mm, ss, wd, *tail = time.gmtime(now)
  421. _cached_formatted_datetime = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
  422. _weekdayname[wd],
  423. day,
  424. _monthname[month],
  425. year,
  426. hh,
  427. mm,
  428. ss,
  429. )
  430. _cached_current_datetime = now
  431. return _cached_formatted_datetime
  432. def _weakref_handle(info: "Tuple[weakref.ref[object], str]") -> None:
  433. ref, name = info
  434. ob = ref()
  435. if ob is not None:
  436. with suppress(Exception):
  437. getattr(ob, name)()
  438. def weakref_handle(
  439. ob: object,
  440. name: str,
  441. timeout: float,
  442. loop: asyncio.AbstractEventLoop,
  443. timeout_ceil_threshold: float = 5,
  444. ) -> Optional[asyncio.TimerHandle]:
  445. if timeout is not None and timeout > 0:
  446. when = loop.time() + timeout
  447. if timeout >= timeout_ceil_threshold:
  448. when = ceil(when)
  449. return loop.call_at(when, _weakref_handle, (weakref.ref(ob), name))
  450. return None
  451. def call_later(
  452. cb: Callable[[], Any],
  453. timeout: float,
  454. loop: asyncio.AbstractEventLoop,
  455. timeout_ceil_threshold: float = 5,
  456. ) -> Optional[asyncio.TimerHandle]:
  457. if timeout is None or timeout <= 0:
  458. return None
  459. now = loop.time()
  460. when = calculate_timeout_when(now, timeout, timeout_ceil_threshold)
  461. return loop.call_at(when, cb)
  462. def calculate_timeout_when(
  463. loop_time: float,
  464. timeout: float,
  465. timeout_ceiling_threshold: float,
  466. ) -> float:
  467. """Calculate when to execute a timeout."""
  468. when = loop_time + timeout
  469. if timeout > timeout_ceiling_threshold:
  470. return ceil(when)
  471. return when
  472. class TimeoutHandle:
  473. """Timeout handle"""
  474. __slots__ = ("_timeout", "_loop", "_ceil_threshold", "_callbacks")
  475. def __init__(
  476. self,
  477. loop: asyncio.AbstractEventLoop,
  478. timeout: Optional[float],
  479. ceil_threshold: float = 5,
  480. ) -> None:
  481. self._timeout = timeout
  482. self._loop = loop
  483. self._ceil_threshold = ceil_threshold
  484. self._callbacks: List[
  485. Tuple[Callable[..., None], Tuple[Any, ...], Dict[str, Any]]
  486. ] = []
  487. def register(
  488. self, callback: Callable[..., None], *args: Any, **kwargs: Any
  489. ) -> None:
  490. self._callbacks.append((callback, args, kwargs))
  491. def close(self) -> None:
  492. self._callbacks.clear()
  493. def start(self) -> Optional[asyncio.TimerHandle]:
  494. timeout = self._timeout
  495. if timeout is not None and timeout > 0:
  496. when = self._loop.time() + timeout
  497. if timeout >= self._ceil_threshold:
  498. when = ceil(when)
  499. return self._loop.call_at(when, self.__call__)
  500. else:
  501. return None
  502. def timer(self) -> "BaseTimerContext":
  503. if self._timeout is not None and self._timeout > 0:
  504. timer = TimerContext(self._loop)
  505. self.register(timer.timeout)
  506. return timer
  507. else:
  508. return TimerNoop()
  509. def __call__(self) -> None:
  510. for cb, args, kwargs in self._callbacks:
  511. with suppress(Exception):
  512. cb(*args, **kwargs)
  513. self._callbacks.clear()
  514. class BaseTimerContext(ContextManager["BaseTimerContext"]):
  515. __slots__ = ()
  516. def assert_timeout(self) -> None:
  517. """Raise TimeoutError if timeout has been exceeded."""
  518. class TimerNoop(BaseTimerContext):
  519. __slots__ = ()
  520. def __enter__(self) -> BaseTimerContext:
  521. return self
  522. def __exit__(
  523. self,
  524. exc_type: Optional[Type[BaseException]],
  525. exc_val: Optional[BaseException],
  526. exc_tb: Optional[TracebackType],
  527. ) -> None:
  528. return
  529. class TimerContext(BaseTimerContext):
  530. """Low resolution timeout context manager"""
  531. __slots__ = ("_loop", "_tasks", "_cancelled", "_cancelling")
  532. def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
  533. self._loop = loop
  534. self._tasks: List[asyncio.Task[Any]] = []
  535. self._cancelled = False
  536. self._cancelling = 0
  537. def assert_timeout(self) -> None:
  538. """Raise TimeoutError if timer has already been cancelled."""
  539. if self._cancelled:
  540. raise asyncio.TimeoutError from None
  541. def __enter__(self) -> BaseTimerContext:
  542. task = asyncio.current_task(loop=self._loop)
  543. if task is None:
  544. raise RuntimeError("Timeout context manager should be used inside a task")
  545. if sys.version_info >= (3, 11):
  546. # Remember if the task was already cancelling
  547. # so when we __exit__ we can decide if we should
  548. # raise asyncio.TimeoutError or let the cancellation propagate
  549. self._cancelling = task.cancelling()
  550. if self._cancelled:
  551. raise asyncio.TimeoutError from None
  552. self._tasks.append(task)
  553. return self
  554. def __exit__(
  555. self,
  556. exc_type: Optional[Type[BaseException]],
  557. exc_val: Optional[BaseException],
  558. exc_tb: Optional[TracebackType],
  559. ) -> Optional[bool]:
  560. enter_task: Optional[asyncio.Task[Any]] = None
  561. if self._tasks:
  562. enter_task = self._tasks.pop()
  563. if exc_type is asyncio.CancelledError and self._cancelled:
  564. assert enter_task is not None
  565. # The timeout was hit, and the task was cancelled
  566. # so we need to uncancel the last task that entered the context manager
  567. # since the cancellation should not leak out of the context manager
  568. if sys.version_info >= (3, 11):
  569. # If the task was already cancelling don't raise
  570. # asyncio.TimeoutError and instead return None
  571. # to allow the cancellation to propagate
  572. if enter_task.uncancel() > self._cancelling:
  573. return None
  574. raise asyncio.TimeoutError from exc_val
  575. return None
  576. def timeout(self) -> None:
  577. if not self._cancelled:
  578. for task in set(self._tasks):
  579. task.cancel()
  580. self._cancelled = True
  581. def ceil_timeout(
  582. delay: Optional[float], ceil_threshold: float = 5
  583. ) -> async_timeout.Timeout:
  584. if delay is None or delay <= 0:
  585. return async_timeout.timeout(None)
  586. loop = asyncio.get_running_loop()
  587. now = loop.time()
  588. when = now + delay
  589. if delay > ceil_threshold:
  590. when = ceil(when)
  591. return async_timeout.timeout_at(when)
  592. class HeadersMixin:
  593. """Mixin for handling headers."""
  594. ATTRS = frozenset(["_content_type", "_content_dict", "_stored_content_type"])
  595. _headers: MultiMapping[str]
  596. _content_type: Optional[str] = None
  597. _content_dict: Optional[Dict[str, str]] = None
  598. _stored_content_type: Union[str, None, _SENTINEL] = sentinel
  599. def _parse_content_type(self, raw: Optional[str]) -> None:
  600. self._stored_content_type = raw
  601. if raw is None:
  602. # default value according to RFC 2616
  603. self._content_type = "application/octet-stream"
  604. self._content_dict = {}
  605. else:
  606. content_type, content_mapping_proxy = parse_content_type(raw)
  607. self._content_type = content_type
  608. # _content_dict needs to be mutable so we can update it
  609. self._content_dict = content_mapping_proxy.copy()
  610. @property
  611. def content_type(self) -> str:
  612. """The value of content part for Content-Type HTTP header."""
  613. raw = self._headers.get(hdrs.CONTENT_TYPE)
  614. if self._stored_content_type != raw:
  615. self._parse_content_type(raw)
  616. assert self._content_type is not None
  617. return self._content_type
  618. @property
  619. def charset(self) -> Optional[str]:
  620. """The value of charset part for Content-Type HTTP header."""
  621. raw = self._headers.get(hdrs.CONTENT_TYPE)
  622. if self._stored_content_type != raw:
  623. self._parse_content_type(raw)
  624. assert self._content_dict is not None
  625. return self._content_dict.get("charset")
  626. @property
  627. def content_length(self) -> Optional[int]:
  628. """The value of Content-Length HTTP header."""
  629. content_length = self._headers.get(hdrs.CONTENT_LENGTH)
  630. return None if content_length is None else int(content_length)
  631. def set_result(fut: "asyncio.Future[_T]", result: _T) -> None:
  632. if not fut.done():
  633. fut.set_result(result)
  634. _EXC_SENTINEL = BaseException()
  635. class ErrorableProtocol(Protocol):
  636. def set_exception(
  637. self,
  638. exc: BaseException,
  639. exc_cause: BaseException = ...,
  640. ) -> None: ... # pragma: no cover
  641. def set_exception(
  642. fut: "asyncio.Future[_T] | ErrorableProtocol",
  643. exc: BaseException,
  644. exc_cause: BaseException = _EXC_SENTINEL,
  645. ) -> None:
  646. """Set future exception.
  647. If the future is marked as complete, this function is a no-op.
  648. :param exc_cause: An exception that is a direct cause of ``exc``.
  649. Only set if provided.
  650. """
  651. if asyncio.isfuture(fut) and fut.done():
  652. return
  653. exc_is_sentinel = exc_cause is _EXC_SENTINEL
  654. exc_causes_itself = exc is exc_cause
  655. if not exc_is_sentinel and not exc_causes_itself:
  656. exc.__cause__ = exc_cause
  657. fut.set_exception(exc)
  658. @functools.total_ordering
  659. class AppKey(Generic[_T]):
  660. """Keys for static typing support in Application."""
  661. __slots__ = ("_name", "_t", "__orig_class__")
  662. # This may be set by Python when instantiating with a generic type. We need to
  663. # support this, in order to support types that are not concrete classes,
  664. # like Iterable, which can't be passed as the second parameter to __init__.
  665. __orig_class__: Type[object]
  666. def __init__(self, name: str, t: Optional[Type[_T]] = None):
  667. # Prefix with module name to help deduplicate key names.
  668. frame = inspect.currentframe()
  669. while frame:
  670. if frame.f_code.co_name == "<module>":
  671. module: str = frame.f_globals["__name__"]
  672. break
  673. frame = frame.f_back
  674. self._name = module + "." + name
  675. self._t = t
  676. def __lt__(self, other: object) -> bool:
  677. if isinstance(other, AppKey):
  678. return self._name < other._name
  679. return True # Order AppKey above other types.
  680. def __repr__(self) -> str:
  681. t = self._t
  682. if t is None:
  683. with suppress(AttributeError):
  684. # Set to type arg.
  685. t = get_args(self.__orig_class__)[0]
  686. if t is None:
  687. t_repr = "<<Unknown>>"
  688. elif isinstance(t, type):
  689. if t.__module__ == "builtins":
  690. t_repr = t.__qualname__
  691. else:
  692. t_repr = f"{t.__module__}.{t.__qualname__}"
  693. else:
  694. t_repr = repr(t)
  695. return f"<AppKey({self._name}, type={t_repr})>"
  696. class ChainMapProxy(Mapping[Union[str, AppKey[Any]], Any]):
  697. __slots__ = ("_maps",)
  698. def __init__(self, maps: Iterable[Mapping[Union[str, AppKey[Any]], Any]]) -> None:
  699. self._maps = tuple(maps)
  700. def __init_subclass__(cls) -> None:
  701. raise TypeError(
  702. "Inheritance class {} from ChainMapProxy "
  703. "is forbidden".format(cls.__name__)
  704. )
  705. @overload # type: ignore[override]
  706. def __getitem__(self, key: AppKey[_T]) -> _T: ...
  707. @overload
  708. def __getitem__(self, key: str) -> Any: ...
  709. def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any:
  710. for mapping in self._maps:
  711. try:
  712. return mapping[key]
  713. except KeyError:
  714. pass
  715. raise KeyError(key)
  716. @overload # type: ignore[override]
  717. def get(self, key: AppKey[_T], default: _S) -> Union[_T, _S]: ...
  718. @overload
  719. def get(self, key: AppKey[_T], default: None = ...) -> Optional[_T]: ...
  720. @overload
  721. def get(self, key: str, default: Any = ...) -> Any: ...
  722. def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any:
  723. try:
  724. return self[key]
  725. except KeyError:
  726. return default
  727. def __len__(self) -> int:
  728. # reuses stored hash values if possible
  729. return len(set().union(*self._maps))
  730. def __iter__(self) -> Iterator[Union[str, AppKey[Any]]]:
  731. d: Dict[Union[str, AppKey[Any]], Any] = {}
  732. for mapping in reversed(self._maps):
  733. # reuses stored hash values if possible
  734. d.update(mapping)
  735. return iter(d)
  736. def __contains__(self, key: object) -> bool:
  737. return any(key in m for m in self._maps)
  738. def __bool__(self) -> bool:
  739. return any(self._maps)
  740. def __repr__(self) -> str:
  741. content = ", ".join(map(repr, self._maps))
  742. return f"ChainMapProxy({content})"
  743. # https://tools.ietf.org/html/rfc7232#section-2.3
  744. _ETAGC = r"[!\x23-\x7E\x80-\xff]+"
  745. _ETAGC_RE = re.compile(_ETAGC)
  746. _QUOTED_ETAG = rf'(W/)?"({_ETAGC})"'
  747. QUOTED_ETAG_RE = re.compile(_QUOTED_ETAG)
  748. LIST_QUOTED_ETAG_RE = re.compile(rf"({_QUOTED_ETAG})(?:\s*,\s*|$)|(.)")
  749. ETAG_ANY = "*"
  750. @attr.s(auto_attribs=True, frozen=True, slots=True)
  751. class ETag:
  752. value: str
  753. is_weak: bool = False
  754. def validate_etag_value(value: str) -> None:
  755. if value != ETAG_ANY and not _ETAGC_RE.fullmatch(value):
  756. raise ValueError(
  757. f"Value {value!r} is not a valid etag. Maybe it contains '\"'?"
  758. )
  759. def parse_http_date(date_str: Optional[str]) -> Optional[datetime.datetime]:
  760. """Process a date string, return a datetime object"""
  761. if date_str is not None:
  762. timetuple = parsedate(date_str)
  763. if timetuple is not None:
  764. with suppress(ValueError):
  765. return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc)
  766. return None
  767. @functools.lru_cache
  768. def must_be_empty_body(method: str, code: int) -> bool:
  769. """Check if a request must return an empty body."""
  770. return (
  771. code in EMPTY_BODY_STATUS_CODES
  772. or method in EMPTY_BODY_METHODS
  773. or (200 <= code < 300 and method in hdrs.METH_CONNECT_ALL)
  774. )
  775. def should_remove_content_length(method: str, code: int) -> bool:
  776. """Check if a Content-Length header should be removed.
  777. This should always be a subset of must_be_empty_body
  778. """
  779. # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-8
  780. # https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4.5-4
  781. return code in EMPTY_BODY_STATUS_CODES or (
  782. 200 <= code < 300 and method in hdrs.METH_CONNECT_ALL
  783. )