_url.py 56 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665
  1. import re
  2. import sys
  3. import warnings
  4. from collections.abc import Mapping, Sequence
  5. from enum import Enum
  6. from functools import _CacheInfo, lru_cache
  7. from ipaddress import ip_address
  8. from typing import (
  9. TYPE_CHECKING,
  10. Any,
  11. NoReturn,
  12. TypedDict,
  13. TypeVar,
  14. Union,
  15. cast,
  16. overload,
  17. )
  18. from urllib.parse import SplitResult, uses_relative
  19. import idna
  20. from multidict import MultiDict, MultiDictProxy, istr
  21. from propcache.api import under_cached_property as cached_property
  22. from ._parse import (
  23. USES_AUTHORITY,
  24. SplitURLType,
  25. make_netloc,
  26. query_to_pairs,
  27. split_netloc,
  28. split_url,
  29. unsplit_result,
  30. )
  31. from ._path import normalize_path, normalize_path_segments
  32. from ._query import (
  33. Query,
  34. QueryVariable,
  35. SimpleQuery,
  36. get_str_query,
  37. get_str_query_from_iterable,
  38. get_str_query_from_sequence_iterable,
  39. )
  40. from ._quoters import (
  41. FRAGMENT_QUOTER,
  42. FRAGMENT_REQUOTER,
  43. PATH_QUOTER,
  44. PATH_REQUOTER,
  45. PATH_SAFE_UNQUOTER,
  46. PATH_UNQUOTER,
  47. QS_UNQUOTER,
  48. QUERY_QUOTER,
  49. QUERY_REQUOTER,
  50. QUOTER,
  51. REQUOTER,
  52. UNQUOTER,
  53. human_quote,
  54. )
  55. try:
  56. from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler
  57. from pydantic.json_schema import JsonSchemaValue
  58. from pydantic_core import core_schema
  59. HAS_PYDANTIC = True
  60. except ImportError:
  61. HAS_PYDANTIC = False
  62. DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21}
  63. USES_RELATIVE = frozenset(uses_relative)
  64. # Special schemes https://url.spec.whatwg.org/#special-scheme
  65. # are not allowed to have an empty host https://url.spec.whatwg.org/#url-representation
  66. SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp"))
  67. # reg-name: unreserved / pct-encoded / sub-delims
  68. # this pattern matches anything that is *not* in those classes. and is only used
  69. # on lower-cased ASCII values.
  70. NOT_REG_NAME = re.compile(
  71. r"""
  72. # any character not in the unreserved or sub-delims sets, plus %
  73. # (validated with the additional check for pct-encoded sequences below)
  74. [^a-z0-9\-._~!$&'()*+,;=%]
  75. |
  76. # % only allowed if it is part of a pct-encoded
  77. # sequence of 2 hex digits.
  78. %(?![0-9a-f]{2})
  79. """,
  80. re.VERBOSE,
  81. )
  82. _T = TypeVar("_T")
  83. if sys.version_info >= (3, 11):
  84. from typing import Self
  85. else:
  86. Self = Any
  87. class UndefinedType(Enum):
  88. """Singleton type for use with not set sentinel values."""
  89. _singleton = 0
  90. UNDEFINED = UndefinedType._singleton
  91. class CacheInfo(TypedDict):
  92. """Host encoding cache."""
  93. idna_encode: _CacheInfo
  94. idna_decode: _CacheInfo
  95. ip_address: _CacheInfo
  96. host_validate: _CacheInfo
  97. encode_host: _CacheInfo
  98. class _InternalURLCache(TypedDict, total=False):
  99. _val: SplitURLType
  100. _origin: "URL"
  101. absolute: bool
  102. hash: int
  103. scheme: str
  104. raw_authority: str
  105. authority: str
  106. raw_user: str | None
  107. user: str | None
  108. raw_password: str | None
  109. password: str | None
  110. raw_host: str | None
  111. host: str | None
  112. host_subcomponent: str | None
  113. host_port_subcomponent: str | None
  114. port: int | None
  115. explicit_port: int | None
  116. raw_path: str
  117. path: str
  118. _parsed_query: list[tuple[str, str]]
  119. query: "MultiDictProxy[str]"
  120. raw_query_string: str
  121. query_string: str
  122. path_qs: str
  123. raw_path_qs: str
  124. raw_fragment: str
  125. fragment: str
  126. raw_parts: tuple[str, ...]
  127. parts: tuple[str, ...]
  128. parent: "URL"
  129. raw_name: str
  130. name: str
  131. raw_suffix: str
  132. suffix: str
  133. raw_suffixes: tuple[str, ...]
  134. suffixes: tuple[str, ...]
  135. def rewrite_module(obj: _T) -> _T:
  136. obj.__module__ = "yarl"
  137. return obj
  138. @lru_cache
  139. def encode_url(url_str: str) -> "URL":
  140. """Parse unencoded URL."""
  141. cache: _InternalURLCache = {}
  142. host: str | None
  143. scheme, netloc, path, query, fragment = split_url(url_str)
  144. if not netloc: # netloc
  145. host = ""
  146. else:
  147. if ":" in netloc or "@" in netloc or "[" in netloc:
  148. # Complex netloc
  149. username, password, host, port = split_netloc(netloc)
  150. else:
  151. username = password = port = None
  152. host = netloc
  153. if host is None:
  154. if scheme in SCHEME_REQUIRES_HOST:
  155. msg = (
  156. "Invalid URL: host is required for "
  157. f"absolute urls with the {scheme} scheme"
  158. )
  159. raise ValueError(msg)
  160. else:
  161. host = ""
  162. host = _encode_host(host, validate_host=False)
  163. # Remove brackets as host encoder adds back brackets for IPv6 addresses
  164. cache["raw_host"] = host[1:-1] if "[" in host else host
  165. cache["explicit_port"] = port
  166. if password is None and username is None:
  167. # Fast path for URLs without user, password
  168. netloc = host if port is None else f"{host}:{port}"
  169. cache["raw_user"] = None
  170. cache["raw_password"] = None
  171. else:
  172. raw_user = REQUOTER(username) if username else username
  173. raw_password = REQUOTER(password) if password else password
  174. netloc = make_netloc(raw_user, raw_password, host, port)
  175. cache["raw_user"] = raw_user
  176. cache["raw_password"] = raw_password
  177. if path:
  178. path = PATH_REQUOTER(path)
  179. if netloc and "." in path:
  180. path = normalize_path(path)
  181. if query:
  182. query = QUERY_REQUOTER(query)
  183. if fragment:
  184. fragment = FRAGMENT_REQUOTER(fragment)
  185. cache["scheme"] = scheme
  186. cache["raw_path"] = "/" if not path and netloc else path
  187. cache["raw_query_string"] = query
  188. cache["raw_fragment"] = fragment
  189. self = object.__new__(URL)
  190. self._scheme = scheme
  191. self._netloc = netloc
  192. self._path = path
  193. self._query = query
  194. self._fragment = fragment
  195. self._cache = cache
  196. return self
  197. @lru_cache
  198. def pre_encoded_url(url_str: str) -> "URL":
  199. """Parse pre-encoded URL."""
  200. self = object.__new__(URL)
  201. val = split_url(url_str)
  202. self._scheme, self._netloc, self._path, self._query, self._fragment = val
  203. self._cache = {}
  204. return self
  205. @lru_cache
  206. def build_pre_encoded_url(
  207. scheme: str,
  208. authority: str,
  209. user: str | None,
  210. password: str | None,
  211. host: str,
  212. port: int | None,
  213. path: str,
  214. query_string: str,
  215. fragment: str,
  216. ) -> "URL":
  217. """Build a pre-encoded URL from parts."""
  218. self = object.__new__(URL)
  219. self._scheme = scheme
  220. if authority:
  221. self._netloc = authority
  222. elif host:
  223. if port is not None:
  224. port = None if port == DEFAULT_PORTS.get(scheme) else port
  225. if user is None and password is None:
  226. self._netloc = host if port is None else f"{host}:{port}"
  227. else:
  228. self._netloc = make_netloc(user, password, host, port)
  229. else:
  230. self._netloc = ""
  231. self._path = path
  232. self._query = query_string
  233. self._fragment = fragment
  234. self._cache = {}
  235. return self
  236. def from_parts_uncached(
  237. scheme: str, netloc: str, path: str, query: str, fragment: str
  238. ) -> "URL":
  239. """Create a new URL from parts."""
  240. self = object.__new__(URL)
  241. self._scheme = scheme
  242. self._netloc = netloc
  243. self._path = path
  244. self._query = query
  245. self._fragment = fragment
  246. self._cache = {}
  247. return self
  248. from_parts = lru_cache(from_parts_uncached)
  249. @rewrite_module
  250. class URL:
  251. # Don't derive from str
  252. # follow pathlib.Path design
  253. # probably URL will not suffer from pathlib problems:
  254. # it's intended for libraries like aiohttp,
  255. # not to be passed into standard library functions like os.open etc.
  256. # URL grammar (RFC 3986)
  257. # pct-encoded = "%" HEXDIG HEXDIG
  258. # reserved = gen-delims / sub-delims
  259. # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
  260. # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
  261. # / "*" / "+" / "," / ";" / "="
  262. # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
  263. # URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]
  264. # hier-part = "//" authority path-abempty
  265. # / path-absolute
  266. # / path-rootless
  267. # / path-empty
  268. # scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
  269. # authority = [ userinfo "@" ] host [ ":" port ]
  270. # userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
  271. # host = IP-literal / IPv4address / reg-name
  272. # IP-literal = "[" ( IPv6address / IPvFuture ) "]"
  273. # IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" )
  274. # IPv6address = 6( h16 ":" ) ls32
  275. # / "::" 5( h16 ":" ) ls32
  276. # / [ h16 ] "::" 4( h16 ":" ) ls32
  277. # / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
  278. # / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
  279. # / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
  280. # / [ *4( h16 ":" ) h16 ] "::" ls32
  281. # / [ *5( h16 ":" ) h16 ] "::" h16
  282. # / [ *6( h16 ":" ) h16 ] "::"
  283. # ls32 = ( h16 ":" h16 ) / IPv4address
  284. # ; least-significant 32 bits of address
  285. # h16 = 1*4HEXDIG
  286. # ; 16 bits of address represented in hexadecimal
  287. # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet
  288. # dec-octet = DIGIT ; 0-9
  289. # / %x31-39 DIGIT ; 10-99
  290. # / "1" 2DIGIT ; 100-199
  291. # / "2" %x30-34 DIGIT ; 200-249
  292. # / "25" %x30-35 ; 250-255
  293. # reg-name = *( unreserved / pct-encoded / sub-delims )
  294. # port = *DIGIT
  295. # path = path-abempty ; begins with "/" or is empty
  296. # / path-absolute ; begins with "/" but not "//"
  297. # / path-noscheme ; begins with a non-colon segment
  298. # / path-rootless ; begins with a segment
  299. # / path-empty ; zero characters
  300. # path-abempty = *( "/" segment )
  301. # path-absolute = "/" [ segment-nz *( "/" segment ) ]
  302. # path-noscheme = segment-nz-nc *( "/" segment )
  303. # path-rootless = segment-nz *( "/" segment )
  304. # path-empty = 0<pchar>
  305. # segment = *pchar
  306. # segment-nz = 1*pchar
  307. # segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
  308. # ; non-zero-length segment without any colon ":"
  309. # pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
  310. # query = *( pchar / "/" / "?" )
  311. # fragment = *( pchar / "/" / "?" )
  312. # URI-reference = URI / relative-ref
  313. # relative-ref = relative-part [ "?" query ] [ "#" fragment ]
  314. # relative-part = "//" authority path-abempty
  315. # / path-absolute
  316. # / path-noscheme
  317. # / path-empty
  318. # absolute-URI = scheme ":" hier-part [ "?" query ]
  319. __slots__ = ("_cache", "_scheme", "_netloc", "_path", "_query", "_fragment")
  320. _cache: _InternalURLCache
  321. _scheme: str
  322. _netloc: str
  323. _path: str
  324. _query: str
  325. _fragment: str
  326. def __new__(
  327. cls,
  328. val: Union[str, SplitResult, "URL", UndefinedType] = UNDEFINED,
  329. *,
  330. encoded: bool = False,
  331. strict: bool | None = None,
  332. ) -> "URL":
  333. if strict is not None: # pragma: no cover
  334. warnings.warn("strict parameter is ignored")
  335. if type(val) is str:
  336. return pre_encoded_url(val) if encoded else encode_url(val)
  337. if type(val) is cls:
  338. return val
  339. if type(val) is SplitResult:
  340. if not encoded:
  341. raise ValueError("Cannot apply decoding to SplitResult")
  342. return from_parts(*val)
  343. if isinstance(val, str):
  344. return pre_encoded_url(str(val)) if encoded else encode_url(str(val))
  345. if val is UNDEFINED:
  346. # Special case for UNDEFINED since it might be unpickling and we do
  347. # not want to cache as the `__set_state__` call would mutate the URL
  348. # object in the `pre_encoded_url` or `encoded_url` caches.
  349. self = object.__new__(URL)
  350. self._scheme = self._netloc = self._path = self._query = self._fragment = ""
  351. self._cache = {}
  352. return self
  353. raise TypeError("Constructor parameter should be str")
  354. @classmethod
  355. def build(
  356. cls,
  357. *,
  358. scheme: str = "",
  359. authority: str = "",
  360. user: str | None = None,
  361. password: str | None = None,
  362. host: str = "",
  363. port: int | None = None,
  364. path: str = "",
  365. query: Query | None = None,
  366. query_string: str = "",
  367. fragment: str = "",
  368. encoded: bool = False,
  369. ) -> "URL":
  370. """Creates and returns a new URL"""
  371. if authority and (user or password or host or port):
  372. raise ValueError(
  373. 'Can\'t mix "authority" with "user", "password", "host" or "port".'
  374. )
  375. if port is not None and not isinstance(port, int):
  376. raise TypeError(f"The port is required to be int, got {type(port)!r}.")
  377. if port and not host:
  378. raise ValueError('Can\'t build URL with "port" but without "host".')
  379. if query and query_string:
  380. raise ValueError('Only one of "query" or "query_string" should be passed')
  381. if (
  382. scheme is None # type: ignore[redundant-expr]
  383. or authority is None # type: ignore[redundant-expr]
  384. or host is None # type: ignore[redundant-expr]
  385. or path is None # type: ignore[redundant-expr]
  386. or query_string is None # type: ignore[redundant-expr]
  387. or fragment is None
  388. ):
  389. raise TypeError(
  390. 'NoneType is illegal for "scheme", "authority", "host", "path", '
  391. '"query_string", and "fragment" args, use empty string instead.'
  392. )
  393. if query:
  394. query_string = get_str_query(query) or ""
  395. if encoded:
  396. return build_pre_encoded_url(
  397. scheme,
  398. authority,
  399. user,
  400. password,
  401. host,
  402. port,
  403. path,
  404. query_string,
  405. fragment,
  406. )
  407. self = object.__new__(URL)
  408. self._scheme = scheme
  409. _host: str | None = None
  410. if authority:
  411. user, password, _host, port = split_netloc(authority)
  412. _host = _encode_host(_host, validate_host=False) if _host else ""
  413. elif host:
  414. _host = _encode_host(host, validate_host=True)
  415. else:
  416. self._netloc = ""
  417. if _host is not None:
  418. if port is not None:
  419. port = None if port == DEFAULT_PORTS.get(scheme) else port
  420. if user is None and password is None:
  421. self._netloc = _host if port is None else f"{_host}:{port}"
  422. else:
  423. self._netloc = make_netloc(user, password, _host, port, True)
  424. path = PATH_QUOTER(path) if path else path
  425. if path and self._netloc:
  426. if "." in path:
  427. path = normalize_path(path)
  428. if path[0] != "/":
  429. msg = (
  430. "Path in a URL with authority should "
  431. "start with a slash ('/') if set"
  432. )
  433. raise ValueError(msg)
  434. self._path = path
  435. if not query and query_string:
  436. query_string = QUERY_QUOTER(query_string)
  437. self._query = query_string
  438. self._fragment = FRAGMENT_QUOTER(fragment) if fragment else fragment
  439. self._cache = {}
  440. return self
  441. def __init_subclass__(cls) -> NoReturn:
  442. raise TypeError(f"Inheriting a class {cls!r} from URL is forbidden")
  443. def __str__(self) -> str:
  444. if not self._path and self._netloc and (self._query or self._fragment):
  445. path = "/"
  446. else:
  447. path = self._path
  448. if (port := self.explicit_port) is not None and port == DEFAULT_PORTS.get(
  449. self._scheme
  450. ):
  451. # port normalization - using None for default ports to remove from rendering
  452. # https://datatracker.ietf.org/doc/html/rfc3986.html#section-6.2.3
  453. host = self.host_subcomponent
  454. netloc = make_netloc(self.raw_user, self.raw_password, host, None)
  455. else:
  456. netloc = self._netloc
  457. return unsplit_result(self._scheme, netloc, path, self._query, self._fragment)
  458. def __repr__(self) -> str:
  459. return f"{self.__class__.__name__}('{str(self)}')"
  460. def __bytes__(self) -> bytes:
  461. return str(self).encode("ascii")
  462. def __eq__(self, other: object) -> bool:
  463. if type(other) is not URL:
  464. return NotImplemented
  465. path1 = "/" if not self._path and self._netloc else self._path
  466. path2 = "/" if not other._path and other._netloc else other._path
  467. return (
  468. self._scheme == other._scheme
  469. and self._netloc == other._netloc
  470. and path1 == path2
  471. and self._query == other._query
  472. and self._fragment == other._fragment
  473. )
  474. def __hash__(self) -> int:
  475. if (ret := self._cache.get("hash")) is None:
  476. path = "/" if not self._path and self._netloc else self._path
  477. ret = self._cache["hash"] = hash(
  478. (self._scheme, self._netloc, path, self._query, self._fragment)
  479. )
  480. return ret
  481. def __le__(self, other: object) -> bool:
  482. if type(other) is not URL:
  483. return NotImplemented
  484. return self._val <= other._val
  485. def __lt__(self, other: object) -> bool:
  486. if type(other) is not URL:
  487. return NotImplemented
  488. return self._val < other._val
  489. def __ge__(self, other: object) -> bool:
  490. if type(other) is not URL:
  491. return NotImplemented
  492. return self._val >= other._val
  493. def __gt__(self, other: object) -> bool:
  494. if type(other) is not URL:
  495. return NotImplemented
  496. return self._val > other._val
  497. def __truediv__(self, name: str) -> "URL":
  498. if not isinstance(name, str):
  499. return NotImplemented # type: ignore[unreachable]
  500. return self._make_child((str(name),))
  501. def __mod__(self, query: Query) -> "URL":
  502. return self.update_query(query)
  503. def __bool__(self) -> bool:
  504. return bool(self._netloc or self._path or self._query or self._fragment)
  505. def __getstate__(self) -> tuple[SplitResult]:
  506. return (tuple.__new__(SplitResult, self._val),)
  507. def __setstate__(
  508. self, state: tuple[SplitURLType] | tuple[None, _InternalURLCache]
  509. ) -> None:
  510. if state[0] is None and isinstance(state[1], dict):
  511. # default style pickle
  512. val = state[1]["_val"]
  513. else:
  514. unused: list[object]
  515. val, *unused = state
  516. self._scheme, self._netloc, self._path, self._query, self._fragment = val
  517. self._cache = {}
  518. def _cache_netloc(self) -> None:
  519. """Cache the netloc parts of the URL."""
  520. c = self._cache
  521. split_loc = split_netloc(self._netloc)
  522. c["raw_user"], c["raw_password"], c["raw_host"], c["explicit_port"] = split_loc
  523. def is_absolute(self) -> bool:
  524. """A check for absolute URLs.
  525. Return True for absolute ones (having scheme or starting
  526. with //), False otherwise.
  527. Is is preferred to call the .absolute property instead
  528. as it is cached.
  529. """
  530. return self.absolute
  531. def is_default_port(self) -> bool:
  532. """A check for default port.
  533. Return True if port is default for specified scheme,
  534. e.g. 'http://python.org' or 'http://python.org:80', False
  535. otherwise.
  536. Return False for relative URLs.
  537. """
  538. if (explicit := self.explicit_port) is None:
  539. # If the explicit port is None, then the URL must be
  540. # using the default port unless its a relative URL
  541. # which does not have an implicit port / default port
  542. return self._netloc != ""
  543. return explicit == DEFAULT_PORTS.get(self._scheme)
  544. def origin(self) -> "URL":
  545. """Return an URL with scheme, host and port parts only.
  546. user, password, path, query and fragment are removed.
  547. """
  548. # TODO: add a keyword-only option for keeping user/pass maybe?
  549. return self._origin
  550. @cached_property
  551. def _val(self) -> SplitURLType:
  552. return (self._scheme, self._netloc, self._path, self._query, self._fragment)
  553. @cached_property
  554. def _origin(self) -> "URL":
  555. """Return an URL with scheme, host and port parts only.
  556. user, password, path, query and fragment are removed.
  557. """
  558. if not (netloc := self._netloc):
  559. raise ValueError("URL should be absolute")
  560. if not (scheme := self._scheme):
  561. raise ValueError("URL should have scheme")
  562. if "@" in netloc:
  563. encoded_host = self.host_subcomponent
  564. netloc = make_netloc(None, None, encoded_host, self.explicit_port)
  565. elif not self._path and not self._query and not self._fragment:
  566. return self
  567. return from_parts(scheme, netloc, "", "", "")
  568. def relative(self) -> "URL":
  569. """Return a relative part of the URL.
  570. scheme, user, password, host and port are removed.
  571. """
  572. if not self._netloc:
  573. raise ValueError("URL should be absolute")
  574. return from_parts("", "", self._path, self._query, self._fragment)
  575. @cached_property
  576. def absolute(self) -> bool:
  577. """A check for absolute URLs.
  578. Return True for absolute ones (having scheme or starting
  579. with //), False otherwise.
  580. """
  581. # `netloc`` is an empty string for relative URLs
  582. # Checking `netloc` is faster than checking `hostname`
  583. # because `hostname` is a property that does some extra work
  584. # to parse the host from the `netloc`
  585. return self._netloc != ""
  586. @cached_property
  587. def scheme(self) -> str:
  588. """Scheme for absolute URLs.
  589. Empty string for relative URLs or URLs starting with //
  590. """
  591. return self._scheme
  592. @cached_property
  593. def raw_authority(self) -> str:
  594. """Encoded authority part of URL.
  595. Empty string for relative URLs.
  596. """
  597. return self._netloc
  598. @cached_property
  599. def authority(self) -> str:
  600. """Decoded authority part of URL.
  601. Empty string for relative URLs.
  602. """
  603. return make_netloc(self.user, self.password, self.host, self.port)
  604. @cached_property
  605. def raw_user(self) -> str | None:
  606. """Encoded user part of URL.
  607. None if user is missing.
  608. """
  609. # not .username
  610. self._cache_netloc()
  611. return self._cache["raw_user"]
  612. @cached_property
  613. def user(self) -> str | None:
  614. """Decoded user part of URL.
  615. None if user is missing.
  616. """
  617. if (raw_user := self.raw_user) is None:
  618. return None
  619. return UNQUOTER(raw_user)
  620. @cached_property
  621. def raw_password(self) -> str | None:
  622. """Encoded password part of URL.
  623. None if password is missing.
  624. """
  625. self._cache_netloc()
  626. return self._cache["raw_password"]
  627. @cached_property
  628. def password(self) -> str | None:
  629. """Decoded password part of URL.
  630. None if password is missing.
  631. """
  632. if (raw_password := self.raw_password) is None:
  633. return None
  634. return UNQUOTER(raw_password)
  635. @cached_property
  636. def raw_host(self) -> str | None:
  637. """Encoded host part of URL.
  638. None for relative URLs.
  639. When working with IPv6 addresses, use the `host_subcomponent` property instead
  640. as it will return the host subcomponent with brackets.
  641. """
  642. # Use host instead of hostname for sake of shortness
  643. # May add .hostname prop later
  644. self._cache_netloc()
  645. return self._cache["raw_host"]
  646. @cached_property
  647. def host(self) -> str | None:
  648. """Decoded host part of URL.
  649. None for relative URLs.
  650. """
  651. if (raw := self.raw_host) is None:
  652. return None
  653. if raw and raw[-1].isdigit() or ":" in raw:
  654. # IP addresses are never IDNA encoded
  655. return raw
  656. return _idna_decode(raw)
  657. @cached_property
  658. def host_subcomponent(self) -> str | None:
  659. """Return the host subcomponent part of URL.
  660. None for relative URLs.
  661. https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
  662. `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`
  663. Examples:
  664. - `http://example.com:8080` -> `example.com`
  665. - `http://example.com:80` -> `example.com`
  666. - `https://127.0.0.1:8443` -> `127.0.0.1`
  667. - `https://[::1]:8443` -> `[::1]`
  668. - `http://[::1]` -> `[::1]`
  669. """
  670. if (raw := self.raw_host) is None:
  671. return None
  672. return f"[{raw}]" if ":" in raw else raw
  673. @cached_property
  674. def host_port_subcomponent(self) -> str | None:
  675. """Return the host and port subcomponent part of URL.
  676. Trailing dots are removed from the host part.
  677. This value is suitable for use in the Host header of an HTTP request.
  678. None for relative URLs.
  679. https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
  680. `IP-literal = "[" ( IPv6address / IPvFuture ) "]"`
  681. https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.3
  682. port = *DIGIT
  683. Examples:
  684. - `http://example.com:8080` -> `example.com:8080`
  685. - `http://example.com:80` -> `example.com`
  686. - `http://example.com.:80` -> `example.com`
  687. - `https://127.0.0.1:8443` -> `127.0.0.1:8443`
  688. - `https://[::1]:8443` -> `[::1]:8443`
  689. - `http://[::1]` -> `[::1]`
  690. """
  691. if (raw := self.raw_host) is None:
  692. return None
  693. if raw[-1] == ".":
  694. # Remove all trailing dots from the netloc as while
  695. # they are valid FQDNs in DNS, TLS validation fails.
  696. # See https://github.com/aio-libs/aiohttp/issues/3636.
  697. # To avoid string manipulation we only call rstrip if
  698. # the last character is a dot.
  699. raw = raw.rstrip(".")
  700. port = self.explicit_port
  701. if port is None or port == DEFAULT_PORTS.get(self._scheme):
  702. return f"[{raw}]" if ":" in raw else raw
  703. return f"[{raw}]:{port}" if ":" in raw else f"{raw}:{port}"
  704. @cached_property
  705. def port(self) -> int | None:
  706. """Port part of URL, with scheme-based fallback.
  707. None for relative URLs or URLs without explicit port and
  708. scheme without default port substitution.
  709. """
  710. if (explicit_port := self.explicit_port) is not None:
  711. return explicit_port
  712. return DEFAULT_PORTS.get(self._scheme)
  713. @cached_property
  714. def explicit_port(self) -> int | None:
  715. """Port part of URL, without scheme-based fallback.
  716. None for relative URLs or URLs without explicit port.
  717. """
  718. self._cache_netloc()
  719. return self._cache["explicit_port"]
  720. @cached_property
  721. def raw_path(self) -> str:
  722. """Encoded path of URL.
  723. / for absolute URLs without path part.
  724. """
  725. return self._path if self._path or not self._netloc else "/"
  726. @cached_property
  727. def path(self) -> str:
  728. """Decoded path of URL.
  729. / for absolute URLs without path part.
  730. """
  731. return PATH_UNQUOTER(self._path) if self._path else "/" if self._netloc else ""
  732. @cached_property
  733. def path_safe(self) -> str:
  734. """Decoded path of URL.
  735. / for absolute URLs without path part.
  736. / (%2F) and % (%25) are not decoded
  737. """
  738. if self._path:
  739. return PATH_SAFE_UNQUOTER(self._path)
  740. return "/" if self._netloc else ""
  741. @cached_property
  742. def _parsed_query(self) -> list[tuple[str, str]]:
  743. """Parse query part of URL."""
  744. return query_to_pairs(self._query)
  745. @cached_property
  746. def query(self) -> "MultiDictProxy[str]":
  747. """A MultiDictProxy representing parsed query parameters in decoded
  748. representation.
  749. Empty value if URL has no query part.
  750. """
  751. return MultiDictProxy(MultiDict(self._parsed_query))
  752. @cached_property
  753. def raw_query_string(self) -> str:
  754. """Encoded query part of URL.
  755. Empty string if query is missing.
  756. """
  757. return self._query
  758. @cached_property
  759. def query_string(self) -> str:
  760. """Decoded query part of URL.
  761. Empty string if query is missing.
  762. """
  763. return QS_UNQUOTER(self._query) if self._query else ""
  764. @cached_property
  765. def path_qs(self) -> str:
  766. """Decoded path of URL with query."""
  767. return self.path if not (q := self.query_string) else f"{self.path}?{q}"
  768. @cached_property
  769. def raw_path_qs(self) -> str:
  770. """Encoded path of URL with query."""
  771. if q := self._query:
  772. return f"{self._path}?{q}" if self._path or not self._netloc else f"/?{q}"
  773. return self._path if self._path or not self._netloc else "/"
  774. @cached_property
  775. def raw_fragment(self) -> str:
  776. """Encoded fragment part of URL.
  777. Empty string if fragment is missing.
  778. """
  779. return self._fragment
  780. @cached_property
  781. def fragment(self) -> str:
  782. """Decoded fragment part of URL.
  783. Empty string if fragment is missing.
  784. """
  785. return UNQUOTER(self._fragment) if self._fragment else ""
  786. @cached_property
  787. def raw_parts(self) -> tuple[str, ...]:
  788. """A tuple containing encoded *path* parts.
  789. ('/',) for absolute URLs if *path* is missing.
  790. """
  791. path = self._path
  792. if self._netloc:
  793. return ("/", *path[1:].split("/")) if path else ("/",)
  794. if path and path[0] == "/":
  795. return ("/", *path[1:].split("/"))
  796. return tuple(path.split("/"))
  797. @cached_property
  798. def parts(self) -> tuple[str, ...]:
  799. """A tuple containing decoded *path* parts.
  800. ('/',) for absolute URLs if *path* is missing.
  801. """
  802. return tuple(UNQUOTER(part) for part in self.raw_parts)
  803. @cached_property
  804. def parent(self) -> "URL":
  805. """A new URL with last part of path removed and cleaned up query and
  806. fragment.
  807. """
  808. path = self._path
  809. if not path or path == "/":
  810. if self._fragment or self._query:
  811. return from_parts(self._scheme, self._netloc, path, "", "")
  812. return self
  813. parts = path.split("/")
  814. return from_parts(self._scheme, self._netloc, "/".join(parts[:-1]), "", "")
  815. @cached_property
  816. def raw_name(self) -> str:
  817. """The last part of raw_parts."""
  818. parts = self.raw_parts
  819. if not self._netloc:
  820. return parts[-1]
  821. parts = parts[1:]
  822. return parts[-1] if parts else ""
  823. @cached_property
  824. def name(self) -> str:
  825. """The last part of parts."""
  826. return UNQUOTER(self.raw_name)
  827. @cached_property
  828. def raw_suffix(self) -> str:
  829. name = self.raw_name
  830. i = name.rfind(".")
  831. return name[i:] if 0 < i < len(name) - 1 else ""
  832. @cached_property
  833. def suffix(self) -> str:
  834. return UNQUOTER(self.raw_suffix)
  835. @cached_property
  836. def raw_suffixes(self) -> tuple[str, ...]:
  837. name = self.raw_name
  838. if name.endswith("."):
  839. return ()
  840. name = name.lstrip(".")
  841. return tuple("." + suffix for suffix in name.split(".")[1:])
  842. @cached_property
  843. def suffixes(self) -> tuple[str, ...]:
  844. return tuple(UNQUOTER(suffix) for suffix in self.raw_suffixes)
  845. def _make_child(self, paths: "Sequence[str]", encoded: bool = False) -> "URL":
  846. """
  847. add paths to self._path, accounting for absolute vs relative paths,
  848. keep existing, but do not create new, empty segments
  849. """
  850. parsed: list[str] = []
  851. needs_normalize: bool = False
  852. for idx, path in enumerate(reversed(paths)):
  853. # empty segment of last is not removed
  854. last = idx == 0
  855. if path and path[0] == "/":
  856. raise ValueError(
  857. f"Appending path {path!r} starting from slash is forbidden"
  858. )
  859. # We need to quote the path if it is not already encoded
  860. # This cannot be done at the end because the existing
  861. # path is already quoted and we do not want to double quote
  862. # the existing path.
  863. path = path if encoded else PATH_QUOTER(path)
  864. needs_normalize |= "." in path
  865. segments = path.split("/")
  866. segments.reverse()
  867. # remove trailing empty segment for all but the last path
  868. parsed += segments[1:] if not last and segments[0] == "" else segments
  869. if (path := self._path) and (old_segments := path.split("/")):
  870. # If the old path ends with a slash, the last segment is an empty string
  871. # and should be removed before adding the new path segments.
  872. old = old_segments[:-1] if old_segments[-1] == "" else old_segments
  873. old.reverse()
  874. parsed += old
  875. # If the netloc is present, inject a leading slash when adding a
  876. # path to an absolute URL where there was none before.
  877. if (netloc := self._netloc) and parsed and parsed[-1] != "":
  878. parsed.append("")
  879. parsed.reverse()
  880. if not netloc or not needs_normalize:
  881. return from_parts(self._scheme, netloc, "/".join(parsed), "", "")
  882. path = "/".join(normalize_path_segments(parsed))
  883. # If normalizing the path segments removed the leading slash, add it back.
  884. if path and path[0] != "/":
  885. path = f"/{path}"
  886. return from_parts(self._scheme, netloc, path, "", "")
  887. def with_scheme(self, scheme: str) -> "URL":
  888. """Return a new URL with scheme replaced."""
  889. # N.B. doesn't cleanup query/fragment
  890. if not isinstance(scheme, str):
  891. raise TypeError("Invalid scheme type")
  892. lower_scheme = scheme.lower()
  893. netloc = self._netloc
  894. if not netloc and lower_scheme in SCHEME_REQUIRES_HOST:
  895. msg = (
  896. "scheme replacement is not allowed for "
  897. f"relative URLs for the {lower_scheme} scheme"
  898. )
  899. raise ValueError(msg)
  900. return from_parts(lower_scheme, netloc, self._path, self._query, self._fragment)
  901. def with_user(self, user: str | None) -> "URL":
  902. """Return a new URL with user replaced.
  903. Autoencode user if needed.
  904. Clear user/password if user is None.
  905. """
  906. # N.B. doesn't cleanup query/fragment
  907. if user is None:
  908. password = None
  909. elif isinstance(user, str):
  910. user = QUOTER(user)
  911. password = self.raw_password
  912. else:
  913. raise TypeError("Invalid user type")
  914. if not (netloc := self._netloc):
  915. raise ValueError("user replacement is not allowed for relative URLs")
  916. encoded_host = self.host_subcomponent or ""
  917. netloc = make_netloc(user, password, encoded_host, self.explicit_port)
  918. return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
  919. def with_password(self, password: str | None) -> "URL":
  920. """Return a new URL with password replaced.
  921. Autoencode password if needed.
  922. Clear password if argument is None.
  923. """
  924. # N.B. doesn't cleanup query/fragment
  925. if password is None:
  926. pass
  927. elif isinstance(password, str):
  928. password = QUOTER(password)
  929. else:
  930. raise TypeError("Invalid password type")
  931. if not (netloc := self._netloc):
  932. raise ValueError("password replacement is not allowed for relative URLs")
  933. encoded_host = self.host_subcomponent or ""
  934. port = self.explicit_port
  935. netloc = make_netloc(self.raw_user, password, encoded_host, port)
  936. return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
  937. def with_host(self, host: str) -> "URL":
  938. """Return a new URL with host replaced.
  939. Autoencode host if needed.
  940. Changing host for relative URLs is not allowed, use .join()
  941. instead.
  942. """
  943. # N.B. doesn't cleanup query/fragment
  944. if not isinstance(host, str):
  945. raise TypeError("Invalid host type")
  946. if not (netloc := self._netloc):
  947. raise ValueError("host replacement is not allowed for relative URLs")
  948. if not host:
  949. raise ValueError("host removing is not allowed")
  950. encoded_host = _encode_host(host, validate_host=True) if host else ""
  951. port = self.explicit_port
  952. netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)
  953. return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
  954. def with_port(self, port: int | None) -> "URL":
  955. """Return a new URL with port replaced.
  956. Clear port to default if None is passed.
  957. """
  958. # N.B. doesn't cleanup query/fragment
  959. if port is not None:
  960. if isinstance(port, bool) or not isinstance(port, int):
  961. raise TypeError(f"port should be int or None, got {type(port)}")
  962. if not (0 <= port <= 65535):
  963. raise ValueError(f"port must be between 0 and 65535, got {port}")
  964. if not (netloc := self._netloc):
  965. raise ValueError("port replacement is not allowed for relative URLs")
  966. encoded_host = self.host_subcomponent or ""
  967. netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port)
  968. return from_parts(self._scheme, netloc, self._path, self._query, self._fragment)
  969. def with_path(
  970. self,
  971. path: str,
  972. *,
  973. encoded: bool = False,
  974. keep_query: bool = False,
  975. keep_fragment: bool = False,
  976. ) -> "URL":
  977. """Return a new URL with path replaced."""
  978. netloc = self._netloc
  979. if not encoded:
  980. path = PATH_QUOTER(path)
  981. if netloc:
  982. path = normalize_path(path) if "." in path else path
  983. if path and path[0] != "/":
  984. path = f"/{path}"
  985. query = self._query if keep_query else ""
  986. fragment = self._fragment if keep_fragment else ""
  987. return from_parts(self._scheme, netloc, path, query, fragment)
  988. @overload
  989. def with_query(self, query: Query) -> "URL": ...
  990. @overload
  991. def with_query(self, **kwargs: QueryVariable) -> "URL": ...
  992. def with_query(self, *args: Any, **kwargs: Any) -> "URL":
  993. """Return a new URL with query part replaced.
  994. Accepts any Mapping (e.g. dict, multidict.MultiDict instances)
  995. or str, autoencode the argument if needed.
  996. A sequence of (key, value) pairs is supported as well.
  997. It also can take an arbitrary number of keyword arguments.
  998. Clear query if None is passed.
  999. """
  1000. # N.B. doesn't cleanup query/fragment
  1001. query = get_str_query(*args, **kwargs) or ""
  1002. return from_parts_uncached(
  1003. self._scheme, self._netloc, self._path, query, self._fragment
  1004. )
  1005. @overload
  1006. def extend_query(self, query: Query) -> "URL": ...
  1007. @overload
  1008. def extend_query(self, **kwargs: QueryVariable) -> "URL": ...
  1009. def extend_query(self, *args: Any, **kwargs: Any) -> "URL":
  1010. """Return a new URL with query part combined with the existing.
  1011. This method will not remove existing query parameters.
  1012. Example:
  1013. >>> url = URL('http://example.com/?a=1&b=2')
  1014. >>> url.extend_query(a=3, c=4)
  1015. URL('http://example.com/?a=1&b=2&a=3&c=4')
  1016. """
  1017. if not (new_query := get_str_query(*args, **kwargs)):
  1018. return self
  1019. if query := self._query:
  1020. # both strings are already encoded so we can use a simple
  1021. # string join
  1022. query += new_query if query[-1] == "&" else f"&{new_query}"
  1023. else:
  1024. query = new_query
  1025. return from_parts_uncached(
  1026. self._scheme, self._netloc, self._path, query, self._fragment
  1027. )
  1028. @overload
  1029. def update_query(self, query: Query) -> "URL": ...
  1030. @overload
  1031. def update_query(self, **kwargs: QueryVariable) -> "URL": ...
  1032. def update_query(self, *args: Any, **kwargs: Any) -> "URL":
  1033. """Return a new URL with query part updated.
  1034. This method will overwrite existing query parameters.
  1035. Example:
  1036. >>> url = URL('http://example.com/?a=1&b=2')
  1037. >>> url.update_query(a=3, c=4)
  1038. URL('http://example.com/?a=3&b=2&c=4')
  1039. """
  1040. in_query: (
  1041. str
  1042. | Mapping[str, QueryVariable]
  1043. | Sequence[tuple[str | istr, SimpleQuery]]
  1044. | None
  1045. )
  1046. if kwargs:
  1047. if args:
  1048. msg = "Either kwargs or single query parameter must be present"
  1049. raise ValueError(msg)
  1050. in_query = kwargs
  1051. elif len(args) == 1:
  1052. in_query = args[0]
  1053. else:
  1054. raise ValueError("Either kwargs or single query parameter must be present")
  1055. if in_query is None:
  1056. query = ""
  1057. elif not in_query:
  1058. query = self._query
  1059. elif isinstance(in_query, Mapping):
  1060. qm: MultiDict[QueryVariable] = MultiDict(self._parsed_query)
  1061. qm.update(in_query)
  1062. query = get_str_query_from_sequence_iterable(qm.items())
  1063. elif isinstance(in_query, str):
  1064. qstr: MultiDict[str] = MultiDict(self._parsed_query)
  1065. qstr.update(query_to_pairs(in_query))
  1066. query = get_str_query_from_iterable(qstr.items())
  1067. elif isinstance(in_query, (bytes, bytearray, memoryview)):
  1068. msg = "Invalid query type: bytes, bytearray and memoryview are forbidden"
  1069. raise TypeError(msg)
  1070. elif isinstance(in_query, Sequence):
  1071. # We don't expect sequence values if we're given a list of pairs
  1072. # already; only mappings like builtin `dict` which can't have the
  1073. # same key pointing to multiple values are allowed to use
  1074. # `_query_seq_pairs`.
  1075. if TYPE_CHECKING:
  1076. in_query = cast(
  1077. Sequence[tuple[Union[str, istr], SimpleQuery]], in_query
  1078. )
  1079. qs: MultiDict[SimpleQuery] = MultiDict(self._parsed_query)
  1080. qs.update(in_query)
  1081. query = get_str_query_from_iterable(qs.items())
  1082. else:
  1083. raise TypeError(
  1084. "Invalid query type: only str, mapping or "
  1085. "sequence of (key, value) pairs is allowed"
  1086. )
  1087. return from_parts_uncached(
  1088. self._scheme, self._netloc, self._path, query, self._fragment
  1089. )
  1090. def without_query_params(self, *query_params: str) -> "URL":
  1091. """Remove some keys from query part and return new URL."""
  1092. params_to_remove = set(query_params) & self.query.keys()
  1093. if not params_to_remove:
  1094. return self
  1095. return self.with_query(
  1096. tuple(
  1097. (name, value)
  1098. for name, value in self.query.items()
  1099. if name not in params_to_remove
  1100. )
  1101. )
  1102. def with_fragment(self, fragment: str | None) -> "URL":
  1103. """Return a new URL with fragment replaced.
  1104. Autoencode fragment if needed.
  1105. Clear fragment to default if None is passed.
  1106. """
  1107. # N.B. doesn't cleanup query/fragment
  1108. if fragment is None:
  1109. raw_fragment = ""
  1110. elif not isinstance(fragment, str):
  1111. raise TypeError("Invalid fragment type")
  1112. else:
  1113. raw_fragment = FRAGMENT_QUOTER(fragment)
  1114. if self._fragment == raw_fragment:
  1115. return self
  1116. return from_parts(
  1117. self._scheme, self._netloc, self._path, self._query, raw_fragment
  1118. )
  1119. def with_name(
  1120. self,
  1121. name: str,
  1122. *,
  1123. keep_query: bool = False,
  1124. keep_fragment: bool = False,
  1125. ) -> "URL":
  1126. """Return a new URL with name (last part of path) replaced.
  1127. Query and fragment parts are cleaned up.
  1128. Name is encoded if needed.
  1129. """
  1130. # N.B. DOES cleanup query/fragment
  1131. if not isinstance(name, str):
  1132. raise TypeError("Invalid name type")
  1133. if "/" in name:
  1134. raise ValueError("Slash in name is not allowed")
  1135. name = PATH_QUOTER(name)
  1136. if name in (".", ".."):
  1137. raise ValueError(". and .. values are forbidden")
  1138. parts = list(self.raw_parts)
  1139. if netloc := self._netloc:
  1140. if len(parts) == 1:
  1141. parts.append(name)
  1142. else:
  1143. parts[-1] = name
  1144. parts[0] = "" # replace leading '/'
  1145. else:
  1146. parts[-1] = name
  1147. if parts[0] == "/":
  1148. parts[0] = "" # replace leading '/'
  1149. query = self._query if keep_query else ""
  1150. fragment = self._fragment if keep_fragment else ""
  1151. return from_parts(self._scheme, netloc, "/".join(parts), query, fragment)
  1152. def with_suffix(
  1153. self,
  1154. suffix: str,
  1155. *,
  1156. keep_query: bool = False,
  1157. keep_fragment: bool = False,
  1158. ) -> "URL":
  1159. """Return a new URL with suffix (file extension of name) replaced.
  1160. Query and fragment parts are cleaned up.
  1161. suffix is encoded if needed.
  1162. """
  1163. if not isinstance(suffix, str):
  1164. raise TypeError("Invalid suffix type")
  1165. if suffix and not suffix[0] == "." or suffix == "." or "/" in suffix:
  1166. raise ValueError(f"Invalid suffix {suffix!r}")
  1167. name = self.raw_name
  1168. if not name:
  1169. raise ValueError(f"{self!r} has an empty name")
  1170. old_suffix = self.raw_suffix
  1171. suffix = PATH_QUOTER(suffix)
  1172. name = name + suffix if not old_suffix else name[: -len(old_suffix)] + suffix
  1173. if name in (".", ".."):
  1174. raise ValueError(". and .. values are forbidden")
  1175. parts = list(self.raw_parts)
  1176. if netloc := self._netloc:
  1177. if len(parts) == 1:
  1178. parts.append(name)
  1179. else:
  1180. parts[-1] = name
  1181. parts[0] = "" # replace leading '/'
  1182. else:
  1183. parts[-1] = name
  1184. if parts[0] == "/":
  1185. parts[0] = "" # replace leading '/'
  1186. query = self._query if keep_query else ""
  1187. fragment = self._fragment if keep_fragment else ""
  1188. return from_parts(self._scheme, netloc, "/".join(parts), query, fragment)
  1189. def join(self, url: "URL") -> "URL":
  1190. """Join URLs
  1191. Construct a full (“absolute”) URL by combining a “base URL”
  1192. (self) with another URL (url).
  1193. Informally, this uses components of the base URL, in
  1194. particular the addressing scheme, the network location and
  1195. (part of) the path, to provide missing components in the
  1196. relative URL.
  1197. """
  1198. if type(url) is not URL:
  1199. raise TypeError("url should be URL")
  1200. scheme = url._scheme or self._scheme
  1201. if scheme != self._scheme or scheme not in USES_RELATIVE:
  1202. return url
  1203. # scheme is in uses_authority as uses_authority is a superset of uses_relative
  1204. if (join_netloc := url._netloc) and scheme in USES_AUTHORITY:
  1205. return from_parts(scheme, join_netloc, url._path, url._query, url._fragment)
  1206. orig_path = self._path
  1207. if join_path := url._path:
  1208. if join_path[0] == "/":
  1209. path = join_path
  1210. elif not orig_path:
  1211. path = f"/{join_path}"
  1212. elif orig_path[-1] == "/":
  1213. path = f"{orig_path}{join_path}"
  1214. else:
  1215. # …
  1216. # and relativizing ".."
  1217. # parts[0] is / for absolute urls,
  1218. # this join will add a double slash there
  1219. path = "/".join([*self.parts[:-1], ""]) + join_path
  1220. # which has to be removed
  1221. if orig_path[0] == "/":
  1222. path = path[1:]
  1223. path = normalize_path(path) if "." in path else path
  1224. else:
  1225. path = orig_path
  1226. return from_parts(
  1227. scheme,
  1228. self._netloc,
  1229. path,
  1230. url._query if join_path or url._query else self._query,
  1231. url._fragment if join_path or url._fragment else self._fragment,
  1232. )
  1233. def joinpath(self, *other: str, encoded: bool = False) -> "URL":
  1234. """Return a new URL with the elements in other appended to the path."""
  1235. return self._make_child(other, encoded=encoded)
  1236. def human_repr(self) -> str:
  1237. """Return decoded human readable string for URL representation."""
  1238. user = human_quote(self.user, "#/:?@[]")
  1239. password = human_quote(self.password, "#/:?@[]")
  1240. if (host := self.host) and ":" in host:
  1241. host = f"[{host}]"
  1242. path = human_quote(self.path, "#?")
  1243. if TYPE_CHECKING:
  1244. assert path is not None
  1245. query_string = "&".join(
  1246. "{}={}".format(human_quote(k, "#&+;="), human_quote(v, "#&+;="))
  1247. for k, v in self.query.items()
  1248. )
  1249. fragment = human_quote(self.fragment, "")
  1250. if TYPE_CHECKING:
  1251. assert fragment is not None
  1252. netloc = make_netloc(user, password, host, self.explicit_port)
  1253. return unsplit_result(self._scheme, netloc, path, query_string, fragment)
  1254. if HAS_PYDANTIC: # pragma: no cover
  1255. # Borrowed from https://docs.pydantic.dev/latest/concepts/types/#handling-third-party-types
  1256. @classmethod
  1257. def __get_pydantic_json_schema__(
  1258. cls, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler
  1259. ) -> JsonSchemaValue:
  1260. field_schema: dict[str, Any] = {}
  1261. field_schema.update(type="string", format="uri")
  1262. return field_schema
  1263. @classmethod
  1264. def __get_pydantic_core_schema__(
  1265. cls, source_type: type[Self] | type[str], handler: GetCoreSchemaHandler
  1266. ) -> core_schema.CoreSchema:
  1267. from_str_schema = core_schema.chain_schema(
  1268. [
  1269. core_schema.str_schema(),
  1270. core_schema.no_info_plain_validator_function(URL),
  1271. ]
  1272. )
  1273. return core_schema.json_or_python_schema(
  1274. json_schema=from_str_schema,
  1275. python_schema=core_schema.union_schema(
  1276. [
  1277. # check if it's an instance first before doing any further work
  1278. core_schema.is_instance_schema(URL),
  1279. from_str_schema,
  1280. ]
  1281. ),
  1282. serialization=core_schema.plain_serializer_function_ser_schema(str),
  1283. )
  1284. _DEFAULT_IDNA_SIZE = 256
  1285. _DEFAULT_ENCODE_SIZE = 512
  1286. @lru_cache(_DEFAULT_IDNA_SIZE)
  1287. def _idna_decode(raw: str) -> str:
  1288. try:
  1289. return idna.decode(raw.encode("ascii"))
  1290. except UnicodeError: # e.g. '::1'
  1291. return raw.encode("ascii").decode("idna")
  1292. @lru_cache(_DEFAULT_IDNA_SIZE)
  1293. def _idna_encode(host: str) -> str:
  1294. try:
  1295. return idna.encode(host, uts46=True).decode("ascii")
  1296. except UnicodeError:
  1297. return host.encode("idna").decode("ascii")
  1298. @lru_cache(_DEFAULT_ENCODE_SIZE)
  1299. def _encode_host(host: str, validate_host: bool) -> str:
  1300. """Encode host part of URL."""
  1301. # If the host ends with a digit or contains a colon, its likely
  1302. # an IP address.
  1303. if host and (host[-1].isdigit() or ":" in host):
  1304. raw_ip, sep, zone = host.partition("%")
  1305. # If it looks like an IP, we check with _ip_compressed_version
  1306. # and fall-through if its not an IP address. This is a performance
  1307. # optimization to avoid parsing IP addresses as much as possible
  1308. # because it is orders of magnitude slower than almost any other
  1309. # operation this library does.
  1310. # Might be an IP address, check it
  1311. #
  1312. # IP Addresses can look like:
  1313. # https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2
  1314. # - 127.0.0.1 (last character is a digit)
  1315. # - 2001:db8::ff00:42:8329 (contains a colon)
  1316. # - 2001:db8::ff00:42:8329%eth0 (contains a colon)
  1317. # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should
  1318. # have been removed before it gets here)
  1319. # Rare IP Address formats are not supported per:
  1320. # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4
  1321. #
  1322. # IP parsing is slow, so its wrapped in an LRU
  1323. try:
  1324. ip = ip_address(raw_ip)
  1325. except ValueError:
  1326. pass
  1327. else:
  1328. # These checks should not happen in the
  1329. # LRU to keep the cache size small
  1330. host = ip.compressed
  1331. if ip.version == 6:
  1332. return f"[{host}%{zone}]" if sep else f"[{host}]"
  1333. return f"{host}%{zone}" if sep else host
  1334. # IDNA encoding is slow, skip it for ASCII-only strings
  1335. if host.isascii():
  1336. # Check for invalid characters explicitly; _idna_encode() does this
  1337. # for non-ascii host names.
  1338. host = host.lower()
  1339. if validate_host and (invalid := NOT_REG_NAME.search(host)):
  1340. value, pos, extra = invalid.group(), invalid.start(), ""
  1341. if value == "@" or (value == ":" and "@" in host[pos:]):
  1342. # this looks like an authority string
  1343. extra = (
  1344. ", if the value includes a username or password, "
  1345. "use 'authority' instead of 'host'"
  1346. )
  1347. raise ValueError(
  1348. f"Host {host!r} cannot contain {value!r} (at position {pos}){extra}"
  1349. ) from None
  1350. return host
  1351. return _idna_encode(host)
  1352. @rewrite_module
  1353. def cache_clear() -> None:
  1354. """Clear all LRU caches."""
  1355. _idna_encode.cache_clear()
  1356. _idna_decode.cache_clear()
  1357. _encode_host.cache_clear()
  1358. @rewrite_module
  1359. def cache_info() -> CacheInfo:
  1360. """Report cache statistics."""
  1361. return {
  1362. "idna_encode": _idna_encode.cache_info(),
  1363. "idna_decode": _idna_decode.cache_info(),
  1364. "ip_address": _encode_host.cache_info(),
  1365. "host_validate": _encode_host.cache_info(),
  1366. "encode_host": _encode_host.cache_info(),
  1367. }
  1368. @rewrite_module
  1369. def cache_configure(
  1370. *,
  1371. idna_encode_size: int | None = _DEFAULT_IDNA_SIZE,
  1372. idna_decode_size: int | None = _DEFAULT_IDNA_SIZE,
  1373. ip_address_size: int | None | UndefinedType = UNDEFINED,
  1374. host_validate_size: int | None | UndefinedType = UNDEFINED,
  1375. encode_host_size: int | None | UndefinedType = UNDEFINED,
  1376. ) -> None:
  1377. """Configure LRU cache sizes."""
  1378. global _idna_decode, _idna_encode, _encode_host
  1379. # ip_address_size, host_validate_size are no longer
  1380. # used, but are kept for backwards compatibility.
  1381. if ip_address_size is not UNDEFINED or host_validate_size is not UNDEFINED:
  1382. warnings.warn(
  1383. "cache_configure() no longer accepts the "
  1384. "ip_address_size or host_validate_size arguments, "
  1385. "they are used to set the encode_host_size instead "
  1386. "and will be removed in the future",
  1387. DeprecationWarning,
  1388. stacklevel=2,
  1389. )
  1390. if encode_host_size is not None:
  1391. for size in (ip_address_size, host_validate_size):
  1392. if size is None:
  1393. encode_host_size = None
  1394. elif encode_host_size is UNDEFINED:
  1395. if size is not UNDEFINED:
  1396. encode_host_size = size
  1397. elif size is not UNDEFINED:
  1398. if TYPE_CHECKING:
  1399. assert isinstance(size, int)
  1400. assert isinstance(encode_host_size, int)
  1401. encode_host_size = max(size, encode_host_size)
  1402. if encode_host_size is UNDEFINED:
  1403. encode_host_size = _DEFAULT_ENCODE_SIZE
  1404. _encode_host = lru_cache(encode_host_size)(_encode_host.__wrapped__)
  1405. _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__)
  1406. _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__)