Ë
    µŒj #  ã                   óø   — d dl Z d dlZd dlZd dlmZmZ d dlmZmZm	Z	m
Z
mZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmZ  ej0                  e«      Z e«       ddd	d
dd
dœZdededefd„Z G d„ de«      Zy)é    N)ÚFutureÚThreadPoolExecutor)	ÚAnyÚAsyncIteratorÚDictÚIteratorÚListÚOptionalÚTupleÚUnionÚcast)ÚDocument)Ú
BaseLoader)Úget_user_agentzJtext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8zen-US,en;q=0.5zhttps://www.google.com/Ú1z
keep-alive)ú
User-AgentÚAcceptzAccept-LanguageÚRefererÚDNTÚ
ConnectionzUpgrade-Insecure-RequestsÚsoupÚurlÚreturnc                 ó  — d|i}| j                  d«      x}r|j                  «       |d<   | j                  dddi¬«      x}r|j                  dd«      |d<   | j                  d	«      x}r|j                  d
d«      |d<   |S )z)Build metadata from BeautifulSoup output.ÚsourceÚtitleÚmetaÚnameÚdescription)ÚattrsÚcontentzNo description found.ÚhtmlÚlangzNo language found.Úlanguage)ÚfindÚget_textÚget)r   r   Úmetadatar   r   r"   s         úy/var/www/html/Fitness-lenito-AI-main/venv/lib/python3.12/site-packages/langchain_community/document_loaders/async_html.pyÚ_build_metadatar*   &   s’   € à˜#ˆ€HØ—	‘	˜'Ó"Ð"€uÐ"Ø!ŸN™NÓ,ˆ�ÑØ—i‘i ¨v°}Ð.E�iÓFÐF€{ÐFØ"-§/¡/°)Ð=TÓ"Uˆ�ÑØ�y‰y˜Ó Ð €tÐ Ø#Ÿx™x¨Ð0DÓEˆ�ÑØ€Oó    c                   óœ  — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 d&dddœdeeee   f   dee   dee	   d	ee   d
e	dee   dede
deeeef      de	de	de	de	fd„Zdedefd„Zededdfd„«       Z	 d'dede
de
dedef
d„Zdedej(                  deeef   fd„Zdee   de	deeeef      fd „Zdee   dee   fd!„Zded"edefd#„Zdee   fd$„Zdee   fd%„Zy)(ÚAsyncHtmlLoaderzLoad `HTML` asynchronously.NTF)Úpreserve_orderÚ	trust_envÚweb_pathÚheader_templateÚ
verify_sslÚproxiesÚautoset_encodingÚencodingÚdefault_parserÚrequests_per_secondÚrequests_kwargsÚraise_for_statusÚignore_load_errorsr.   r/   c                ót  — t        |t        «      r	|g| _        nt        |t        «      r|| _        |xs t        }|j                  d«      s	 ddlm}  |«       j                  |d<   t        j                  «       | _        t        |«      | j                  _        || j                  _        |r%| j                  j$                  j'                  |«       || _        || _        |	xs i | _        |
| _        || _        || _        || _        || _        || _        y# t        $ r t        j                  d«       Y ŒÏw xY w)zInitialize with a webpage path.r   r   )Ú	UserAgentzxfake_useragent not found, using default user agent.To get a realistic header for requests, `pip install fake_useragent`.N)Ú
isinstanceÚstrÚ	web_pathsr	   Údefault_header_templater'   Úfake_useragentr<   ÚrandomÚImportErrorÚloggerÚinfoÚrequestsÚSessionÚsessionÚdictÚheadersÚverifyr3   Úupdater7   r6   r8   r9   r4   r5   r:   r.   r/   )Úselfr0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r.   r/   rJ   r<   s                   r)   Ú__init__zAsyncHtmlLoader.__init__5   s  € ô, �h¤Ô$Ø&˜ZˆD�NÜ˜¤$Ô'Ø%ˆDŒNà!Ò<Ô%<ˆØ�{‰{˜<Ô(ð	Ý4á(1«×(:Ñ(:�˜Ñ%ô  ×'Ñ'Ó)ˆŒÜ# G›}ˆ�‰ÔØ(ˆ�‰ÔáØ�L‰L× Ñ ×'Ñ'¨Ô0à#6ˆÔ Ø,ˆÔØ.Ò4°"ˆÔØ 0ˆÔØ 0ˆÔØ ˆŒØ"4ˆÔØ,ˆÔà"ˆ�øô/ ò Ü—‘ð4öðús   ÁD ÄD7Ä6D7r   r   c                 ó   — | j                   r(	  | j                  j                  |fi | j                  ¤ŽS  | j                  j                  |fi | j                  ¤ŽS # t        $ r(}t        j                  t        |«      «       Y d }~y d }~ww xY w©N)r:   rH   r'   r8   Ú	ExceptionÚwarningsÚwarnr>   )rM   r   Úes      r)   Ú_fetch_valid_connection_docsz,AsyncHtmlLoader._fetch_valid_connection_docso   s{   € Ø×"Ò"ðØ'�t—|‘|×'Ñ'¨ÑD¨t×/CÑ/CÑDÐDð
  ˆt�|‰|×Ñ Ñ< t×';Ñ';Ñ<Ð<øô	 ò Ü—‘œc !›fÔ%Üûðús   Ž&A Á	BÁ%BÂBÚparserc                 óT   — g d¢}| |vr t        ddj                  |«      z   dz   «      ‚y)z#Check that parser is valid for bs4.)úhtml.parserÚlxmlÚxmlzlxml-xmlÚhtml5libz`parser` must be one of z, Ú.N)Ú
ValueErrorÚjoin)rV   Úvalid_parserss     r)   Ú_check_parserzAsyncHtmlLoader._check_parsery   s:   € ò OˆØ˜Ñ&ÜØ*¨T¯Y©Y°}Ó-EÑEÈÑKóð ð 'r+   ÚretriesÚcooldownÚbackoffc              ƒ   ó’  K  — t        j                  | j                  ¬«      4 ƒd {  –—† }t        |«      D ]Æ  }	 t	        d| j
                  j                  | j
                  j                  j                  «       dœ| j                  ¤Ž}| j
                  j                  sd|d<    |j                  |fi |¤Ž4 ƒd {  –—† }	 |j                  «       ƒ d {  –—† }	|	cd d d «      ƒd {  –—†  c cd d d «      ƒd {  –—†  S  d d d «      ƒd {  –—†  t-        d«      ‚7 Œó7 Œa7 ŒJ# t        $ r t        j                  d|› �«       d}	Y Œmw xY w7 Œc7 ŒR# 1 ƒd {  –—†7  sw Y   nxY w�Œ.# t         j                   t"        f$ r¤}
||dz
  k(  r?| j$                  r3t        j'                  d|› d	|› d
�«       Y d }
~
 d d d «      ƒd {  –—†7   y||dz
  k(  r‚ t        j'                  d|› d|dz   › d|› d|
› d�	«       t)        j*                  |||z  z  «      ƒ d {  –—†7   Y d }
~
�Œæd }
~
ww xY w7 �Œ# 1 ƒd {  –—†7  sw Y   t-        d«      ‚xY w­w)N)r/   )rJ   ÚcookiesFÚsslzFailed to decode content from Ú é   zError fetching z after z	 retries.z with attempt Ú/z: z. Retrying...zretry count exceeded© )ÚaiohttpÚClientSessionr/   ÚrangerI   rH   rJ   re   Úget_dictr8   rK   r'   ÚtextÚUnicodeDecodeErrorrD   ÚerrorÚClientConnectionErrorÚTimeoutErrorr:   ÚwarningÚasyncioÚsleepr]   )rM   r   ra   rb   rc   rH   ÚiÚkwargsÚresponsero   rT   s              r)   Ú_fetchzAsyncHtmlLoader._fetch‚   s2  è ø€ ô ×(Ñ(°4·>±>×BÕBÀgÜ˜7–^�ðCÜ#'ð $Ø $§¡× 4Ñ 4Ø $§¡× 4Ñ 4× =Ñ =Ó ?ñ$ð ×.Ñ.ñ$�Fð
  Ÿ<™<×.Ò.Ø(-˜˜u™Ø*˜wŸ{™{Øñ à ÷ ò  ð "ð&Ø)1¯©«×#8˜Dð  $÷ ÷  ò  ÷ C×BÑBØ#÷ C×Bô@ Ð/Ó0Ð0ðA Cøð øð
 $9ùÜ1ò &Ü"ŸL™LÐ+IÈ#ÈÐ)OÔPØ#%šDð&úð øð Cø÷ ÷  ñ  úñ  øô  ×5Ñ5´|ÐDò CØ˜G a™KÒ'¨D×,CÒ,CÜŸ™¨¸¸¸WÀWÀIÈYÐ'WÔXÜ!÷/ C×BÑBð0 ˜g¨™kÒ)ØäŸ™Ø-¨c¨U°.Ø  1™u˜g Q w i¨r°!°°MðCôô &Ÿm™m¨H°wÀ±zÑ,AÓB×B×BûðCúð) Cù×B×BÐBô@ Ð/Ó0Ð0üs;  ‚%I§D¨I«H(»B E%Â;D
Â<E%Â?EÃD!ÃD
ÃD!ÃEÃE%Ã&E

Ã'E%Ã+H(Ã-IÃ9EÃ:IÃ?H(Ä IÄH%ÄIÄE%ÄD!Ä!#E	ÅEÅE	ÅEÅ
E%ÅIÅE ÅEÅE ÅE%Å#H(Å%H"Å>0HÆ.H(Æ3IÆ>GÆ?IÇAHÈH
ÈHÈH(ÈH"È"H(È%IÈ(IÈ.H1È/IÈ6IÚ	semaphorec              ƒ   ó®   K  — |4 ƒd {  –—†  || j                  |«      ƒ d {  –—† fcd d d «      ƒd {  –—†  S 7 Œ07 Œ7 Œ	# 1 ƒd {  –—†7  sw Y   y xY w­wrP   )rz   )rM   r   r{   s      r)   Ú_fetch_with_rate_limitz&AsyncHtmlLoader._fetch_with_rate_limit§   s;   è ø€ ÷ ’9Ø˜dŸk™k¨#Ó.×.Ð.÷ —9’9øØ.øð ø—9—9‘9üsL   ‚A‰:ŠA�A £<¤A ¨A´>µA¼A ¾AÁ AÁA	ÁAÁAÚurlsc           	     óH  K  — t        j                  | j                  «      }|D �cg c]'  }t        j                  | j	                  ||«      «      ‘Œ) }}	 ddlm} |r ||ddd¬«      D ]  }|ƒ d {  –—† ­–— Œ y |j                  |ddd¬«      D ]  }|ƒ d {  –—† ­–— Œ y c c}w 7 Œ77 Œ# t        $ ri t        j                  d«       |r't        j                  |Ž ƒ d {  –—†7  D ]  }|­–— Œ	 Y y t        j                  |«      D ]  }|ƒ d {  –—†7  ­–— Œ Y y w xY w­w)Nr   )Útqdm_asynciozFetching pagesTrh   )ÚdescÚasciiÚminintervalz2For better logging of progress, `pip install tqdm`)ru   Ú	Semaphorer7   Úcreate_taskr}   Útqdm.asyncior€   Úas_completedrC   rR   rS   Úgather)	rM   r~   r.   r{   r   Útasksr€   ÚtaskÚresults	            r)   Ú_lazy_fetch_allzAsyncHtmlLoader._lazy_fetch_all­   s#  è ø€ ô ×%Ñ% d×&>Ñ&>Ó?ˆ	ñ ó
á�ô ×Ñ × ;Ñ ;¸CÀÓ KÕLØð 	ð 
ð	%Ý1áÙ(ØÐ 0¸È!÷�Dð !%Ÿ*Ô$ñð
 )×5Ñ5ØÐ 0¸È!ð 6ö �Dð !%Ÿ*Ô$ñùò
ð %øð
 %ùÜò 	%Ü�M‰MÐNÔOÙÜ$+§N¡N°EÐ$:×:Ó:�FØ ”Lò ;ô $×0Ñ0°Ö7�DØ $Ÿ*˜*Ô$ò 8ð	%üsˆ   ‚$D"¦,B$ÁD"ÁB- Á1B)Á2	B- Á;D"Á<B- ÂB+Â	B- Â#D"Â)B- Â+B- Â-5DÃ"C%Ã#DÃ2D"Ã4DÄDÄ
DÄD"ÄDÄD"c              ƒ   óp   K  — | j                  |d«      2 ��cg c3 d{  –—† \  }}|‘Œ7 Œ
6 c}}S c c}}w ­w)z/Fetch all urls concurrently with rate limiting.TN)rŒ   )rM   r~   Ú_Údocs       r)   Ú	fetch_allzAsyncHtmlLoader.fetch_allË   s2   è ø€ à(,×(<Ñ(<¸TÀ4Ô(H×IÔI™f˜a ’ÐIøÓIÐIùÓIùs$   ‚6˜0š+ž)Ÿ+¢0©+«0¬
6ro   c                 ó´   — ddl m} |j                  d«      rd}n| j                  }| j	                  |«        |||«      }t        ||«      }t        ||¬«      S )Nr   )ÚBeautifulSoupz.xmlrZ   )Úpage_contentr(   )Úbs4r’   Úendswithr6   r`   r*   r   )rM   r   ro   r’   rV   r   r(   s          r)   Ú_to_documentzAsyncHtmlLoader._to_documentÏ   sT   € Ý%à�<‰<˜ÔØ‰Fà×(Ñ(ˆFØ×Ñ˜6Ô"Ù˜T 6Ó*ˆÜ" 4¨Ó-ˆÜ T°HÔ=Ð=r+   c              #   ó  K  — 	 t        j                  «        t        d¬«      5 }|j                  t         j                  | j                  | j                  «      «      }|j                  «       }ddd«       t        t        t        t           «      «      D ]&  \  }}| j                  | j                  |   |«      –— Œ( y# 1 sw Y   ŒSxY w# t        $ r1 t        j                  | j                  | j                  «      «      }Y Œ�w xY w­w)ú+Lazy load text from the url(s) in web_path.rh   )Úmax_workersN)ru   Úget_running_loopr   ÚsubmitÚrunr�   r?   r‹   ÚRuntimeErrorÚ	enumerater   r	   r>   r–   )rM   ÚexecutorÚfutureÚresultsrw   ro   s         r)   Ú	lazy_loadzAsyncHtmlLoader.lazy_loadÛ   sÌ   è ø€ ð	Bä×$Ñ$Ô&ô $°Õ2°hØ,4¯O©OÜ—K‘KØ—N‘N 4§>¡>Ó2ó-�ð !Ÿ-™-›/�÷ 3ô !¤¤d¬3¡i°Ó!9Ö:‰GˆAˆtØ×#Ñ# D§N¡N°1Ñ$5°tÓ<Ó<ñ ;÷ 3Ð2ûô ò 	BÜ—k‘k $§.¡.°·±Ó"@ÓAŠGð	BüsA   ‚D	„ C ¤A
C Á.C Á6A
D	Ã C	ÃC Ã7DÄD	ÄDÄD	c                ó¢   K  — | j                  | j                  | j                  «      2 3 d{  –—† \  }}| j                  ||«      ­–— Œ"7 Œ6 y­w)r˜   N)rŒ   r?   r.   r–   )rM   r   ro   s      r)   Ú
alazy_loadzAsyncHtmlLoader.alazy_loadï   sN   è ø€ à#×3Ñ3Ø�N‰N˜D×/Ñ/ô 
÷ 	/‘)�#�tð ×#Ñ# C¨Ó.Ô.ð	/øñ  
ùs%   ‚'A©A­A®A±AÁAÁA)
NTNTNrX   é   NFF)é   r¥   g      ø?)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r>   r	   r
   rI   ÚboolÚintr   r   rN   rU   Ústaticmethodr`   Úfloatrz   ru   r„   r   r}   r   rŒ   r�   r   r–   r   r¢   r¤   rj   r+   r)   r-   r-   2   sÿ  „ Ù%ð
 +/Ø%)Ø"&Ø!%Ø"&Ø+Ø#$Ø48Ø!&Ø#(ð8#ð  $Øò8#à˜˜T #™Y˜Ñ'ð8#ð " $™ð8#ð ˜T‘Nð	8#ð
 ˜$‘ð8#ð ð8#ð ˜3‘-ð8#ð ð8#ð !ð8#ð " $ s¨C x¡.Ñ1ð8#ð ð8#ð !ð8#ð ð8#ð ó8#ðt=°ð =¸ó =ð ð˜cð  dò ó ðð ORñ#1Øð#1Ø!$ð#1Ø47ð#1ØFKð#1à	ó#1ðJ/Øð/Ø#*×#4Ñ#4ð/à	ˆs�Cˆx‰ó/ð%Ø˜‘Ið%Ø/3ð%à	�u˜S #˜X‘Ñ	'ó%ð<J D¨¡Ið J°$°s±)ó Jð
> ð 
>¨3ð 
>°8ó 
>ð=˜8 HÑ-ó =ð(/ -°Ñ"9ô /r+   r-   ) ru   ÚloggingrR   Úconcurrent.futuresr   r   Útypingr   r   r   r   r	   r
   r   r   r   rk   rF   Úlangchain_core.documentsr   Ú)langchain_community.document_loaders.baser   Ú$langchain_community.utils.user_agentr   Ú	getLoggerr§   rD   r@   r>   rI   r*   r-   rj   r+   r)   Ú<module>r¶      s�   ðÛ Û Û ß 9÷
÷ 
õ 
ó Û Ý -å @Ý ?à	ˆ×	Ñ	˜8Ó	$€ñ !Ó"ðà'Ø(ØØØ!$ñ	Ð ð	˜#ð 	 Cð 	¨Dó 	ôB/�jõ B/r+   