§
    šŠtj¿  ã                  ó²   — d dl mZ d dlmZ d dlmZmZmZmZm	Z	m
Z
mZmZ d dlmZ d dlmZ erd dlmZ d dlmZmZ  G d„ d	e¦  «        Zdd„Zdd„Zdd„ZdS )é    )Úannotations)ÚPath)ÚTYPE_CHECKINGÚAnyÚIteratorÚListÚOptionalÚSequenceÚTupleÚUnion)ÚDocument)Ú
BaseLoader©ÚNavigableString©ÚCommentÚTagc                  ó6   — e Zd ZdZ	 	 	 	 	 ddd„Zdd„Zdd„ZdS )ÚReadTheDocsLoaderz+Load `ReadTheDocs` documentation directory.N©z*.htmz*.htmlç      ð?ÚpathúUnion[str, Path]ÚencodingúOptional[str]ÚerrorsÚcustom_html_tagúOptional[Tuple[str, dict]]ÚpatternsúSequence[str]Úexclude_links_ratioÚfloatÚkwargsúOptional[Any]c                ó(  — 	 ddl m} n# t          $ r t          d¦  «        ‚w xY w	  |	 	 di |¤Ž}	n"# t          $ r}
t	          d¦  «        |
‚d}
~
ww xY wt          |¦  «        | _        || _        || _        || _	        || _
        || _        || _        dS )	uZ  
        Initialize ReadTheDocsLoader

        The loader loops over all files under `path` and extracts the actual content of
        the files by retrieving main html tags. Default main html tags include
        `<main id="main-content>`, <`div role="main>`, and `<article role="main">`. You
        can also define your own html tags by passing custom_html_tag, e.g.
        `("div", "class=main")`. The loader iterates html tags with the order of
        custom html tags (if exists) and default html tags. If any of the tags is not
        empty, the loop will break and retrieve the content out of that tag.

        Args:
            path: The location of pulled readthedocs folder.
            encoding: The encoding with which to open the documents.
            errors: Specify how encoding and decoding errors are to be handledâ€”this
                cannot be used in binary mode.
            custom_html_tag: Optional custom html tag to retrieve the content from
                files.
            patterns: The file patterns to load, passed to `glob.rglob`.
            exclude_links_ratio: The ratio of links:content to exclude pages from.
                This is to reduce the frequency at which index pages make their
                way into retrieved results. Recommended: 0.5
            kwargs: named arguments passed to `bs4.BeautifulSoup`.
        r   ©ÚBeautifulSoupzWCould not import python packages. Please install it with `pip install beautifulsoup4`. ú6<html><body>Parser builder library test.</body></html>úhtml.parserz"Parsing kwargs do not appear validN)r(   r)   )Úbs4r'   ÚImportErrorÚ	ExceptionÚ
ValueErrorr   Ú	file_pathr   r   r   r   Ú	bs_kwargsr!   )Úselfr   r   r   r   r   r!   r#   r'   Ú_Úes              ún/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langchain_community/document_loaders/readthedocs.pyÚ__init__zReadTheDocsLoader.__init__   sõ   € ðD	Ø)Ð)Ð)Ð)Ð)Ð)Ð)øÝð 	ð 	ð 	ÝðHñô ð ð	øøøð	JØ�ØHØðð ð ðð ˆAˆAøõ
 ð 	Jð 	Jð 	JÝÐAÑBÔBÈÐIøøøøð	Jøøøõ ˜d™œˆŒØ ˆŒØˆŒØ.ˆÔØ ˆŒØˆŒØ#6ˆÔ Ð Ð s   ‚	 ‰#§
2 ²
A¼AÁAÚreturnúIterator[Document]c           	   #  ó|  K  — | j         D ]±}| j                             |¦  «        D ]”}|                     ¦   «         rŒt	          || j        | j        ¬¦  «        5 }|                      |                     ¦   «         ¦  «        }ddd¦  «         n# 1 swxY w Y   t          |dt          |¦  «        i¬¦  «        V — Œ•Œ²dS )zA lazy loader for Documents.)r   r   NÚsource)Úpage_contentÚmetadata)r   r.   ÚrglobÚis_dirÚopenr   r   Ú_clean_dataÚreadr   Ústr)r0   Úfile_patternÚpÚfÚtexts        r3   Ú	lazy_loadzReadTheDocsLoader.lazy_loadM   s  è è € à œMð 	Oð 	OˆLØ”^×)Ò)¨,Ñ7Ô7ð Oð O�Ø—8’8‘:”:ð ØÝ˜! d¤m¸D¼KÐHÑHÔHð 6ÈAØ×+Ò+¨A¯FªF©H¬HÑ5Ô5�Dð6ð 6ð 6ñ 6ô 6ð 6ð 6ð 6ð 6ð 6ð 6øøøð 6ð 6ð 6ð 6å¨D¸HÅcÈ!ÁfÄfÐ;MÐNÑNÔNÐNÐNÐNÐNðOð	Oð 	Os   Á(BÂBÂBÚdatar@   c                óœ  — ddl m}  ||dfi | j        ¤Ž}dddifdddifg}| j        �|                     | j        ¦  «         d }|d d d	…         D ]\  }}|                     ||¦  «        }|� nŒ |�(t          |¦  «        | j        k    rt          |¦  «        }nd
}d 	                    d„ | 
                    d¦  «        D ¦   «         ¦  «        S )Nr   r&   r)   ÚdivÚroleÚmainÚidzmain-contentéÿÿÿÿÚ ú
c                ó   — g | ]}|¯|‘ŒS © rP   )Ú.0Úts     r3   ú
<listcomp>z1ReadTheDocsLoader._clean_data.<locals>.<listcomp>s   s   € Ð;Ð;Ð; ¸Ð;˜!Ð;Ð;Ð;ó    )r*   r'   r/   r   ÚappendÚfindÚ_get_link_ratior!   Ú_get_clean_textÚjoinÚsplit)	r0   rF   r'   ÚsoupÚ	html_tagsÚelementÚtagÚattrsrD   s	            r3   r>   zReadTheDocsLoader._clean_dataW   s  € Ø%Ð%Ð%Ð%Ð%Ð%àˆ}˜T =ÐCÐC°D´NÐCÐCˆð �V˜VÐ$Ð%Ø�d˜NÐ+Ð,ð
ˆ	ð
 ÔÐ+Ø×Ò˜TÔ1Ñ2Ô2Ð2àˆð $ D D b Dœ/ð 	ð 	‰JˆC�Ø—i’i  UÑ+Ô+ˆGàÐ"Ø�ð #ð Ð¥?°7Ñ#;Ô#;¸tÔ?WÒ#WÐ#WÝ" 7Ñ+Ô+ˆDˆDàˆDà�yŠyÐ;Ð; T§Z¢Z°Ñ%5Ô%5Ð;Ñ;Ô;Ñ<Ô<Ð<rT   )NNNr   r   )r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   )r5   r6   )rF   r@   r5   r@   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r4   rE   r>   rP   rT   r3   r   r      sq   € € € € € Ø5Ð5ð
 #'Ø $Ø6:Ø"5Ø%(ð97ð 97ð 97ð 97ð 97ðvOð Oð Oð Oð=ð =ð =ð =ð =ð =rT   r   r]   r   r5   r@   c                ó\   — g d¢}g d¢}t          | ||¦  «        }|                     ¦   «         S )zMReturns cleaned text with newlines preserved and irrelevant elements removed.)ÚscriptÚnoscriptÚcanvasÚmetaÚsvgÚmapÚareaÚaudior8   ÚtrackÚvideoÚembedÚobjectÚparamÚpictureÚiframeÚframeÚframesetÚnoframesÚappletÚformÚbuttonÚselectÚbaseÚstyleÚimg)rB   rH   ÚulÚolÚliÚh1Úh2Úh3Úh4Úh5Úh6ÚpreÚtableÚtr)Ú_process_elementÚstrip)r]   Úelements_to_skipÚnewline_elementsrD   s       r3   rX   rX   v   sI   € ðð ð Ðð:ð ð Ðõ" ˜GÐ%5Ð7GÑHÔH€DØ�:Š:‰<Œ<ÐrT   Úsectionr"   c                ó  — |                       d¦  «        }d                     d„ | j        D ¦   «         ¦  «        }t          |¦  «        dk    rdS d                     d„ |D ¦   «         ¦  «        }t          |¦  «        t          |¦  «        z  S )NÚarM   c              3  ó4   K  — | ]}t          |¦  «        V — Œd S ©N)r@   )rQ   Úss     r3   ú	<genexpr>z"_get_link_ratio.<locals>.<genexpr>¬   s(   è è € ÐBÐB A�˜Q™œÐBÐBÐBÐBÐBÐBrT   r   c              3  óz   K  — | ]6}|j         D ],}|¯t          |j                             ¦   «         ¦  «        V — Œ-Œ7d S r’   )Ústringsr@   Ústringr‹   )rQ   Úlinkr—   s      r3   r”   z"_get_link_ratio.<locals>.<genexpr>°   sk   è è € ð ð àØ”lðð ð Øð	ÝˆFŒM×ÒÑ!Ô!Ñ"Ô"ðð ð ð ð ð ð rT   )Úfind_allrY   Ústripped_stringsÚlen)rŽ   ÚlinksÚ
total_textÚ	link_texts       r3   rW   rW   ª   s–   € Ø×Ò˜SÑ!Ô!€EØ—’ÐBÐB¨Ô)AÐBÑBÔBÑBÔB€JÝ
ˆ:�„˜!ÒÐØˆqà—’ð ð àðñ ô ñ ô €Iõ ˆy‰>Œ>�C 
™OœOÑ+Ð+rT   ú$Union[Tag, NavigableString, Comment]rŒ   ú	List[str]r�   c                ó`  ‡‡‡‡‡— ddl mŠ ddlmŠmŠ t          | dd¦  «        }t          | ‰¦  «        s|‰v rdS t          | ‰¦  «        r| S |dk    rdS |‰v r-d                     ˆˆˆˆˆfd	„| j        D ¦   «         ¦  «        dz   S d                     ˆˆˆˆˆfd
„| j        D ¦   «         ¦  «        S )zq
    Traverse through HTML tree recursively to preserve newline and skip
    unwanted (code/binary) elements
    r   r   r   ÚnameNrM   ÚbrrN   c              3  ó`   •K  — | ](}t          |‰‰‰f¦  «        ¯t          |‰‰¦  «        V — Œ)d S r’   ©Ú
isinstancerŠ   ©rQ   Úchildr   r   r   rŒ   r�   s     €€€€€r3   r”   z#_process_element.<locals>.<genexpr>Î   s\   øè è € ð ð àÝ˜e c¨?¸GÐ%DÑEÔEðÝ  Ð(8Ð:JÑKÔKðð ð ð ð ð rT   c              3  ó`   •K  — | ](}t          |‰‰‰f¦  «        ¯t          |‰‰¦  «        V — Œ)d S r’   r¥   r§   s     €€€€€r3   r”   z#_process_element.<locals>.<genexpr>Ö   s\   øè è € ð 
ð 
àÝ˜% # ¸Ð!@ÑAÔAð
Ý˜UÐ$4Ð6FÑGÔGð
ð 
ð 
ð 
ð 
ð 
rT   )	r*   r   Úbs4.elementr   r   Úgetattrr¦   rY   Úchildren)r]   rŒ   r�   Útag_namer   r   r   s    `` @@@r3   rŠ   rŠ   ¹   sK  øøøøø€ ð $Ð#Ð#Ð#Ð#Ð#Ø(Ð(Ð(Ð(Ð(Ð(Ð(Ð(å�w ¨Ñ-Ô-€HÝ�'˜7Ñ#Ô#ð 
 xÐ3CÐ'CÐ'CØˆrÝ	�G˜_Ñ	-Ô	-ð 
ØˆØ	�TÒ	Ð	ØˆtØ	Ð%Ð	%Ð	%à�GŠGð ð ð ð ð ð ð ð à$Ô-ðñ ô ñ ô ð
 ñð	
ð �wŠwð 
ð 
ð 
ð 
ð 
ð 
ð 
ð 
à Ô)ð
ñ 
ô 
ñ 
ô 
ð 	
rT   N)r]   r   r5   r@   )rŽ   r   r5   r"   )r]   rŸ   rŒ   r    r�   r    r5   r@   )Ú
__future__r   Úpathlibr   Útypingr   r   r   r   r	   r
   r   r   Úlangchain_core.documentsr   Ú)langchain_community.document_loaders.baser   r*   r   rª   r   r   r   rX   rW   rŠ   rP   rT   r3   ú<module>r³      s5  ðØ "Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ð Ð Ø WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ WÐ Wà -Ð -Ð -Ð -Ð -Ð -à @Ð @Ð @Ð @Ð @Ð @àð )Ø#Ð#Ð#Ð#Ð#Ð#Ø(Ð(Ð(Ð(Ð(Ð(Ð(Ð(ðd=ð d=ð d=ð d=ð d=˜
ñ d=ô d=ð d=ðN1ð 1ð 1ð 1ðh,ð ,ð ,ð ,ð!
ð !
ð !
ð !
ð !
ð !
rT   