Ë
    µŒjî;  ã                   óô   — d Z ddlZddlZddlZddlmZmZmZmZm	Z	m
Z
mZmZ ddlZddlZddlmZ ddlmZ ddlmZ ddlmZ  ej.                  e«      Z e«       dd	d
ddddœZdededefd„Z G d„ de«      Zy)zWeb base loader class.é    N)ÚAnyÚAsyncIteratorÚDictÚIteratorÚListÚOptionalÚSequenceÚUnion)Ú
deprecated)ÚDocument)Ú
BaseLoader)Úget_user_agentzJtext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8zen-US,en;q=0.5zhttps://www.google.com/Ú1z
keep-alive)ú
User-AgentÚAcceptzAccept-LanguageÚRefererÚDNTÚ
ConnectionzUpgrade-Insecure-RequestsÚsoupÚurlÚreturnc                 ó  — d|i}| j                  d«      x}r|j                  «       |d<   | j                  dddi¬«      x}r|j                  dd«      |d<   | j                  d	«      x}r|j                  d
d«      |d<   |S )z)Build metadata from BeautifulSoup output.ÚsourceÚtitleÚmetaÚnameÚdescription)ÚattrsÚcontentzNo description found.ÚhtmlÚlangzNo language found.Úlanguage)ÚfindÚget_textÚget)r   r   Úmetadatar   r   r    s         úw/var/www/html/Fitness-lenito-AI-main/venv/lib/python3.12/site-packages/langchain_community/document_loaders/web_base.pyÚ_build_metadatar(      s’   € à˜#ˆ€HØ—	‘	˜'Ó"Ð"€uÐ"Ø!ŸN™NÓ,ˆ�ÑØ—i‘i ¨v°}Ð.E�iÓFÐF€{ÐFØ"-§/¡/°)Ð=TÓ"Uˆ�ÑØ�y‰y˜Ó Ð €tÐ Ø#Ÿx™x¨Ð0DÓEˆ�ÑØ€Oó    c            &       ó–  — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d2dddœdeeee   f   dee   de	d	ee   d
e	de	dee   dee   de
dedeeeef      de	deeeef      deeeef      dede	de	ddf$d„Zedefd„«       Z	 d3dede
de
dedef
d„Zdedej&                  defd „Zd!ee   defd"„Zed#eddfd$„«       Z	 d4d%ed!ee   d#eedf   dee   fd&„Zd4d!ee   d#eedf   dee   fd'„Z	 d4d!ee   d#eedf   dee   fd(„Z	 	 d5ded#eedf   dee   defd)„Zd4d#eedf   defd*„Zdee   fd+„Z de!e   fd,„Z" e#d-d.d/¬0«      dee   fd1„«       Z$y)6ÚWebBaseLoaderaQ  
    WebBaseLoader document loader integration

    Setup:
        Install ``langchain_community``.

        .. code-block:: bash

            pip install -U langchain_community

    Instantiate:
        .. code-block:: python

            from langchain_community.document_loaders import WebBaseLoader

            loader = WebBaseLoader(
                web_path = "https://www.espn.com/"
                # header_template = None,
                # verify_ssl = True,
                # proxies = None,
                # continue_on_failure = False,
                # autoset_encoding = True,
                # encoding = None,
                # web_paths = (),
                # requests_per_second = 2,
                # default_parser = "html.parser",
                # requests_kwargs = None,
                # raise_for_status = False,
                # bs_get_text_kwargs = None,
                # bs_kwargs = None,
                # session = None,
                # show_progress = True,
                # trust_env = False,
            )

    Lazy load:
        .. code-block:: python

            docs = []
            for doc in loader.lazy_load():
                docs.append(doc)
            print(docs[0].page_content[:100])
            print(docs[0].metadata)

        .. code-block:: python

            ESPN - Serving Sports Fans. Anytime. Anywhere.

            {'source': 'https://www.espn.com/', 'title': 'ESPN - Serving Sports Fans. Anytime. Anywhere.', 'description': 'Visit ESPN for live scores, highlights and sports news. Stream exclusive games on ESPN+ and play fantasy sports.', 'language': 'en'}


    Async load:
        .. code-block:: python

            docs = []
            async for doc in loader.alazy_load():
                docs.append(doc)
            print(docs[0].page_content[:100])
            print(docs[0].metadata)

        .. code-block:: python

            ESPN - Serving Sports Fans. Anytime. Anywhere.

            {'source': 'https://www.espn.com/', 'title': 'ESPN - Serving Sports Fans. Anytime. Anywhere.', 'description': 'Visit ESPN for live scores, highlights and sports news. Stream exclusive games on ESPN+ and play fantasy sports.', 'language': 'en'}

    .. versionchanged:: 0.3.14

        Deprecated ``aload`` (which was not async) and implemented a native async
        ``alazy_load``. Expand below for more details.

        .. dropdown:: How to update ``aload``

            Instead of using ``aload``, you can use ``load`` for synchronous loading or
            ``alazy_load`` for asynchronous lazy loading.

            Example using ``load`` (synchronous):

            .. code-block:: python

                docs: List[Document] = loader.load()

            Example using ``alazy_load`` (asynchronous):

            .. code-block:: python

                docs: List[Document] = []
                async for doc in loader.alazy_load():
                    docs.append(doc)

            This is in preparation for accommodating an asynchronous ``aload`` in the
            future:

            .. code-block:: python

                docs: List[Document] = await loader.aload()

    NTF)Úshow_progressÚ	trust_envÚweb_pathÚheader_templateÚ
verify_sslÚproxiesÚcontinue_on_failureÚautoset_encodingÚencodingÚ	web_pathsÚrequests_per_secondÚdefault_parserÚrequests_kwargsÚraise_for_statusÚbs_get_text_kwargsÚ	bs_kwargsÚsessionr,   r-   r   c                ó8  — |r|rt        d«      ‚|rt        |«      | _        n^t        |t        «      r	|g| _        nEt        |t
        «      rt        |«      | _        n$t        dt        |«      › dt        |«      › d�«      ‚|	| _        |
| _	        |xs i | _
        || _        || _        |xs i | _        |xs i | _        |r|| _        n“t!        j"                  «       }|xs t$        j'                  «       }|j)                  d«      s	 ddlm}  |«       j.                  |d<   t7        |«      |_        ||_        |r|j<                  j?                  |«       || _        || _         || _!        || _"        || _#        y	# t0        $ r t2        j5                  d«       Y Œxw xY w)
aØ  Initialize loader.

        Args:
            web_paths: Web paths to load from.
            requests_per_second: Max number of concurrent requests to make.
            default_parser: Default parser to use for BeautifulSoup.
            requests_kwargs: kwargs for requests
            raise_for_status: Raise an exception if http status code denotes an error.
            bs_get_text_kwargs: kwargs for beatifulsoup4 get_text
            bs_kwargs: kwargs for beatifulsoup4 web page parsing
            show_progress: Show progress bar when loading pages.
            trust_env: set to True if using proxy to make web requests, for example
                using http(s)_proxy environment variables. Defaults to False.
        zmReceived web_path and web_paths. Only one can be specified. web_path is deprecated, web_paths should be used.z+web_path must be str or Sequence[str] got (z*) or web_paths must be Sequence[str] got (Ú)r   r   )Ú	UserAgentzxfake_useragent not found, using default user agent.To get a realistic header for requests, `pip install fake_useragent`.N)$Ú
ValueErrorÚlistr5   Ú
isinstanceÚstrr	   Ú	TypeErrorÚtyper6   r7   r8   r9   r,   r:   r;   r<   ÚrequestsÚSessionÚdefault_header_templateÚcopyr%   Úfake_useragentr?   ÚrandomÚImportErrorÚloggerÚinfoÚdictÚheadersÚverifyr1   Úupdater2   r3   r4   r-   )Úselfr.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r,   r-   r?   s                      r'   Ú__init__zWebBaseLoader.__init__Ž   s’  € ñH ™	ÜðDóð ñ Ü! )›_ˆD�NÜ˜¤#Ô&Ø&˜ZˆD�NÜ˜¤(Ô+Ü! (›^ˆD�NäØ=¼dÀ8»nÐ=Mð N9Ü9=¸i»Ð8IÈðLóð ð $7ˆÔ Ø,ˆÔØ.Ò4°"ˆÔØ 0ˆÔØ*ˆÔØ"4Ò":¸ˆÔØ"š bˆŒÙØ"ˆD�Lä×&Ñ&Ó(ˆGØ-ÒOÔ1H×1MÑ1MÓ1OˆOØ"×&Ñ& |Ô4ð	Ý8á4=³K×4FÑ4F�O LÑ1ô # ?Ó3ˆGŒOØ'ˆGŒNÙØ—‘×&Ñ& wÔ/Ø"ˆDŒLØ#6ˆÔ Ø 0ˆÔØ ˆŒØ"ˆ�øô #ò Ü—K‘Kð8öðús   ÄE8 Å8FÆFc                 óf   — t        | j                  «      dkD  rt        d«      ‚| j                  d   S )Né   zMultiple webpaths found.r   )Úlenr5   r@   )rS   s    r'   r.   zWebBaseLoader.web_pathã   s.   € äˆt�~‰~Ó Ò"ÜÐ7Ó8Ð8Ø�~‰~˜aÑ Ð r)   r   ÚretriesÚcooldownÚbackoffc              ƒ   óÜ  K  — t        j                  | j                  ¬«      4 ƒd {  –—† }t        |«      D ]á  }	 t	        | j
                  j                  | j
                  j                  j                  «       ¬«      }| j
                  j                  sd|d<    |j                  |fi | j                  |z  ¤Ž4 ƒd {  –—† }| j                  r|j                  «        |j                  «       ƒ d {  –—† cd d d «      ƒd {  –—†  c cd d d «      ƒd {  –—†  S  d d d «      ƒd {  –—†  t'        d«      ‚7 �Œ7 Œ{7 ŒI7 Œ;7 Œ*# 1 ƒd {  –—†7  sw Y   nxY w�Œ!# t         j                  $ r]}	||dz
  k(  r‚ t        j!                  d|› d|dz   › d|› d	|	› d
�	«       t#        j$                  |||z  z  «      ƒ d {  –—†7   Y d }	~	�ŒŒd }	~	ww xY w7 Œ¤# 1 ƒd {  –—†7  sw Y   t'        d«      ‚xY w­w)N)r-   )rP   ÚcookiesFÚsslrV   úError fetching z with attempt Ú/z: z. Retrying...zretry count exceeded)ÚaiohttpÚClientSessionr-   ÚrangerO   r<   rP   r\   Úget_dictrQ   r%   r8   r9   ÚtextÚClientConnectionErrorrM   ÚwarningÚasyncioÚsleepr@   )
rS   r   rX   rY   rZ   r<   ÚiÚkwargsÚresponseÚes
             r'   Ú_fetchzWebBaseLoader._fetché   sÀ  è ø€ ô ×(Ñ(°4·>±>×BÕBÀgÜ˜7–^�ðCÜ#'Ø $§¡× 4Ñ 4Ø $§¡× 4Ñ 4× =Ñ =Ó ?ô$�Fð  Ÿ<™<×.Ò.Ø(-˜˜u™à*˜wŸ{™{Øñ Ø $× 4Ñ 4°vÑ =÷ ò  à!Ø×0Ò0Ø$×5Ñ5Ô7Ø%-§]¡]£_×4÷ ÷  ò  ÷ C×BÑBØ#÷ C×Bô2 Ð/Ó0Ð0ð3 Cùð øð
  5øð øð Cø÷ ÷  ñ  úñ  øô ×4Ñ4ò CØ˜G a™KÒ'ØäŸ™Ø-¨c¨U°.Ø  1™u˜g Q w i¨r°!°°MðCôô &Ÿm™m¨H°wÀ±zÑ,AÓB×B×BûðCúð! Cø×B×BÐBô2 Ð/Ó0Ð0üsù   ‚%G,§D6¨G,«G»BEÂ=D9
Â>EÃ0EÃ1D;
Ã2EÃ5EÄD=
ÄEÄGÄG,ÄD?ÄG,ÄGÄG,Ä&GÄ'G,Ä9EÄ;EÄ=EÄ?G,ÅEÅE
ÅEÅEÅGÅGÅ+AGÆ7F:
Æ8GÆ=GÇGÇGÇG,ÇG)ÇGÇG)ÇG,Ú	semaphorec              ƒ   óz  K  — |4 ƒd {  –—†  	 | j                  |«      ƒ d {  –—† cd d d «      ƒd {  –—†  S 7 Œ/7 Œ7 Œ	# t        $ r[}| j                  r/t        j	                  d|› d�«       Y d }~d d d «      ƒd {  –—†7   yt        j                  d|› d�«       |‚d }~ww xY w# 1 ƒd {  –—†7  sw Y   y xY w­w)Nr^   z*, skipping due to continue_on_failure=TrueÚ za and aborting, use continue_on_failure=True to continue loading urls after encountering an error.)rm   Ú	Exceptionr2   rM   rf   Ú	exception)rS   r   rn   rl   s       r'   Ú_fetch_with_rate_limitz$WebBaseLoader._fetch_with_rate_limit  s­   è ø€ ÷ ’9ðØ!Ÿ[™[¨Ó-×-÷ —9’9øà-øð ùô ò Ø×+Ò+Ü—N‘NØ)¨#¨ð /4ð 5ôó ÷ —9‘9ô × Ñ Ø% c Uð +Lð Lôð �ûðú÷ —9‘9üsƒ   ‚B;‰9ŠB;�B&�?£;¤?§B;³=´B;»?½B;¿	B#Á%BÁ-B&Á1B;Á<A?Á=B;ÂBÂB#Â#B&Â&B8Â,B/Â-B8Â4B;Úurlsc              ƒ   óÞ  K  — t        j                  | j                  «      }g }|D ]8  }t        j                  | j	                  ||«      «      }|j                  |«       Œ: 	 | j                  r"ddlm}  |j                  |ddddœŽƒ d{  –—† S t        j                  |Ž ƒ d{  –—† S 7 Œ7 Œ# t        $ r3 t        j                  d«       t        j                  |Ž ƒ d{  –—†7  cY S w xY w­w)	z/Fetch all urls concurrently with rate limiting.r   )Útqdm_asynciozFetching pagesTrV   )ÚdescÚasciiÚminintervalNz2For better logging of progress, `pip install tqdm`)rg   Ú	Semaphorer6   Úensure_futurers   Úappendr,   Útqdm.asynciorv   ÚgatherrL   ÚwarningsÚwarn)rS   rt   rn   Útasksr   Útaskrv   s          r'   Ú	fetch_allzWebBaseLoader.fetch_all  sØ   è ø€ ä×%Ñ% d×&>Ñ&>Ó?ˆ	ØˆÛˆCÜ×(Ñ(¨×)DÑ)DÀSÈ)Ó)TÓUˆDØ�L‰L˜Õð ð	0Ø×!Ò!Ý5à0˜\×0Ñ0ØÐ!1¸È1ò÷ ð ô %Ÿ^™^¨UÐ3×3Ð3ð	øð 4ùÜò 	0Ü�M‰MÐNÔOÜ Ÿ™¨Ð/×/Ð/Ò/ð	0üsf   ‚AC-Á")B. ÂB*ÂB. ÂC-ÂB. Â%B,Â&B. Â)C-Â*B. Â,B. Â.3C*Ã!C$Ã"C*Ã'C-Ã)C*Ã*C-Úparserc                 óT   — g d¢}| |vr t        ddj                  |«      z   dz   «      ‚y)z#Check that parser is valid for bs4.)úhtml.parserÚlxmlÚxmlzlxml-xmlÚhtml5libz`parser` must be one of z, Ú.N)r@   Újoin)r„   Úvalid_parserss     r'   Ú_check_parserzWebBaseLoader._check_parser.  s:   € ò OˆØ˜Ñ&ÜØ*¨T¯Y©Y°}Ó-EÑEÈÑKóð ð 'r)   Úresultsc                 óò   — ddl m} g }t        |«      D ]`  \  }}||   }|€1|j                  d«      rd}n| j                  }| j                  |«       |j                   |||fi | j                  ¤Ž«       Œb |S )z0Unpack fetch results into BeautifulSoup objects.r   ©ÚBeautifulSoupú.xmlrˆ   )Úbs4r‘   Ú	enumerateÚendswithr7   r�   r|   r;   )	rS   rŽ   rt   r„   r‘   Úfinal_resultsri   Úresultr   s	            r'   Ú_unpack_fetch_resultsz#WebBaseLoader._unpack_fetch_results7  s}   € õ 	&àˆÜ" 7Ö+‰IˆAˆvØ�q‘'ˆCØˆ~Ø—<‘< Ô'Ø"‘Fà!×0Ñ0�FØ×"Ñ" 6Ô*Ø× Ñ ¡¨v°vÑ!PÀÇÁÑ!PÕQð ,ð Ðr)   c                 ór   — t        j                  | j                  |«      «      }| j                  |||¬«      S )z2Fetch all urls, then return soups for all results.©r„   )rg   Úrunrƒ   r˜   ©rS   rt   r„   rŽ   s       r'   Ú
scrape_allzWebBaseLoader.scrape_allI  s1   € ä—+‘+˜dŸn™n¨TÓ2Ó3ˆØ×)Ñ)¨'°4ÀÐ)ÓGÐGr)   c              ƒ   óh   K  — | j                  |«      ƒ d{  –—† }| j                  |||¬«      S 7 Œ­w)z8Async fetch all urls, then return soups for all results.Nrš   )rƒ   r˜   rœ   s       r'   Úascrape_allzWebBaseLoader.ascrape_allN  s6   è ø€ ð Ÿ™ tÓ,×,ˆØ×)Ñ)¨'°4ÀÐ)ÓGÐGð -ús   ‚2—0˜2c                 ó   — ddl m} |€ |j                  d«      rd}n| j                  }| j	                  |«        | j
                  j                  |fi | j                  ¤Ž}| j                  r|j                  «        | j                  �| j                  |_	        n| j                  r|j                  |_	         ||j                  |fi |xs i ¤ŽS )Nr   r�   r’   rˆ   )r“   r‘   r•   r7   r�   r<   r%   r8   r9   r4   r3   Úapparent_encodingrd   )rS   r   r„   r;   r‘   Úhtml_docs         r'   Ú_scrapezWebBaseLoader._scrapeU  s³   € õ 	&àˆ>Ø�|‰|˜FÔ#Ø‘à×,Ñ,�à×Ñ˜6Ô"à#�4—<‘<×#Ñ# CÑ@¨4×+?Ñ+?Ñ@ˆØ× Ò Ø×%Ñ%Ô'à�=‰=Ð$Ø $§¡ˆHÕØ×"Ò"Ø (× :Ñ :ˆHÔÙ˜XŸ]™]¨FÑH°y²ÀBÑHÐHr)   c                 óR   — | j                  | j                  || j                  ¬«      S )z?Scrape data from webpage and return it in BeautifulSoup format.)r„   r;   )r£   r.   r;   )rS   r„   s     r'   ÚscrapezWebBaseLoader.scrapeo  s!   € ð �|‰|˜DŸM™M°&ÀDÇNÁNˆ|ÓSÐSr)   c              #   óÖ   K  — | j                   D ]V  }| j                  || j                  ¬«      } |j                  di | j                  ¤Ž}t        ||«      }t        ||¬«      –— ŒX y­w)z+Lazy load text from the url(s) in web_path.)r;   ©Úpage_contentr&   N© )r5   r£   r;   r$   r:   r(   r   )rS   Úpathr   rd   r&   s        r'   Ú	lazy_loadzWebBaseLoader.lazy_loadt  s\   è ø€ à—N”NˆDØ—<‘< °·±�<Ó?ˆDØ �4—=‘=Ñ; 4×#:Ñ#:Ñ;ˆDÜ& t¨TÓ2ˆHÜ¨°xÔ@Ó@ñ	 #ùs   ‚A'A)c                ó  K  — | j                  | j                  «      ƒ d{  –—† }t        | j                  |«      D ]=  \  }} |j                  di | j                  ¤Ž}t        ||«      }t        ||¬«      ­–— Œ? y7 Œ[­w)z1Async lazy load text from the url(s) in web_path.Nr§   r©   )rŸ   r5   Úzipr$   r:   r(   r   )rS   rŽ   rª   r   rd   r&   s         r'   Ú
alazy_loadzWebBaseLoader.alazy_load|  sm   è ø€ à×(Ñ(¨¯©Ó8×8ˆÜ˜dŸn™n¨gÖ6‰JˆD�$Ø �4—=‘=Ñ; 4×#:Ñ#:Ñ;ˆDÜ& t¨TÓ2ˆHÜ¨°xÔ@Ô@ñ 7ð 9ús   ‚A?¡A=¢AA?z0.3.14z1.0z«See API reference for updated usage: https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.web_base.WebBaseLoader.html)ÚsinceÚremovalÚmessagec                 ó  — | j                  | j                  «      }g }t        | j                  |«      D ]I  \  }} |j                  di | j                  ¤Ž}t        ||«      }|j                  t        ||¬«      «       ŒK |S )z9Load text from the urls in web_path async into Documents.r§   r©   )r�   r5   r­   r$   r:   r(   r|   r   )rS   rŽ   Údocsrª   r   rd   r&   s          r'   ÚaloadzWebBaseLoader.aload„  ss   € ð —/‘/ $§.¡.Ó1ˆØˆÜ˜dŸn™n¨gÖ6‰JˆD�$Ø �4—=‘=Ñ; 4×#:Ñ#:Ñ;ˆDÜ& t¨TÓ2ˆHØ�K‰Kœ¨d¸XÔFÕGð 7ð
 ˆr)   )rp   NTNFTNr©   é   r†   NFNNN)é   rµ   g      ø?)N)NN)%Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   rC   r	   r   rO   ÚboolÚintr   r   rT   Úpropertyr.   Úfloatrm   rg   rz   rs   r   rƒ   Ústaticmethodr�   r˜   r�   rŸ   r£   r¥   r   r   r«   r   r®   r   r´   r©   r)   r'   r+   r+   *   s_  „ ñaðJ /1Ø*.ØØ"&Ø$)Ø!%Ø"&Ø#%Ø#$Ø+Ø48Ø!&Ø7;Ø.2Øð!S#ð$ #Øò'S#à˜˜X c™]Ð*Ñ+ðS#ð " $™ðS#ð ð	S#ð
 ˜$‘ðS#ð "ðS#ð ðS#ð ˜3‘-ðS#ð ˜C‘=ðS#ð !ðS#ð ðS#ð " $ s¨C x¡.Ñ1ðS#ð ðS#ð % T¨#¨s¨(¡^Ñ4ðS#ð ˜D  c ™NÑ+ðS#ð  ð!S#ð$ ð%S#ð& ð'S#ð( 
ó)S#ðj ð!˜#ò !ó ð!ð ORñ1Øð1Ø!$ð1Ø47ð1ØFKð1à	ó1ð<ØðØ#*×#4Ñ#4ðà	óð&0 D¨¡Ið 0°#ó 0ð( ð˜cð  dò ó ðð IMñØðØ"& s¡)ðØ5:¸3À¸9Ñ5Eðà	ˆc‰óñ$H˜t C™yð H°%¸¸T¸	Ñ2Bð HÈdÐSVÉió Hð ;?ñHØ˜‘IðHØ',¨S°$¨YÑ'7ðHà	ˆc‰óHð $(Ø$(ñ	IàðIð �c˜4�iÑ ðIð ˜D‘>ð	Ið
 
óIñ4T˜U 3¨ 9Ñ-ð T¸ó Tð
A˜8 HÑ-ó AðA -°Ñ"9ó Añ ØØðUô	ð
�t˜H‘~ò 
óñ
r)   r+   )rº   rg   Úloggingr   Útypingr   r   r   r   r   r   r	   r
   r`   rF   Úlangchain_core._apir   Úlangchain_core.documentsr   Ú)langchain_community.document_loaders.baser   Ú$langchain_community.utils.user_agentr   Ú	getLoggerr·   rM   rH   rC   rO   r(   r+   r©   r)   r'   Ú<module>rÇ      sŠ   ðÙ ã Û Û ß V× VÓ Vã Û Ý *Ý -å @Ý ?à	ˆ×	Ñ	˜8Ó	$€ñ !Ó"ðà'Ø(ØØØ!$ñ	Ð ð	˜#ð 	 Cð 	¨Dó 	ôl�Jõ lr)   