
    dh                        S SK Jr  S SKJr  S SKJrJrJrJrJ	r	J
r
JrJr  S SKJr  S SKJr  \(       a  S SKJr  S SKJrJr   " S S	\5      rSS
 jrSS jr        SS jrg)    )annotations)Path)TYPE_CHECKINGAnyIteratorListOptionalSequenceTupleUnion)Document)
BaseLoaderNavigableStringCommentTagc                  ^    \ rS rSrSr     S             S	S jjrS
S jrSS jrSrg)ReadTheDocsLoader   z+Load `ReadTheDocs` documentation directory.Nc                     SSK Jn   U"   S0 UD6n	[        U5      U l        X l        X0l        X@l	        XPl
        Xpl        X`l        g! [         a    [        S5      ef = f! [         a  n
[	        S5      U
eSn
A
ff = f)u  
Initialize ReadTheDocsLoader

The loader loops over all files under `path` and extracts the actual content of
the files by retrieving main html tags. Default main html tags include
`<main id="main-content>`, <`div role="main>`, and `<article role="main">`. You
can also define your own html tags by passing custom_html_tag, e.g.
`("div", "class=main")`. The loader iterates html tags with the order of
custom html tags (if exists) and default html tags. If any of the tags is not
empty, the loop will break and retrieve the content out of that tag.

Args:
    path: The location of pulled readthedocs folder.
    encoding: The encoding with which to open the documents.
    errors: Specify how encoding and decoding errors are to be handled—this
        cannot be used in binary mode.
    custom_html_tag: Optional custom html tag to retrieve the content from
        files.
    patterns: The file patterns to load, passed to `glob.rglob`.
    exclude_links_ratio: The ratio of links:content to exclude pages from.
        This is to reduce the frequency at which index pages make their
        way into retrieved results. Recommended: 0.5
    kwargs: named arguments passed to `bs4.BeautifulSoup`.
r   BeautifulSoupzWCould not import python packages. Please install it with `pip install beautifulsoup4`. z"Parsing kwargs do not appear validN)z6<html><body>Parser builder library test.</body></html>html.parser)bs4r   ImportError	Exception
ValueErrorr   	file_pathencodingerrorscustom_html_tagpatterns	bs_kwargsexclude_links_ratio)selfpathr    r!   r"   r#   r%   kwargsr   _es              h/var/www/html/shao/venv/lib/python3.13/site-packages/langchain_community/document_loaders/readthedocs.py__init__ReadTheDocsLoader.__init__   s    D	)	JH A d . #6 +  	H 	  	JABI	Js"   A 
A! A!
A<+A77A<c           	   #    #    U R                    H  nU R                  R                  U5       Hw  nUR                  5       (       a  M  [	        X R
                  U R                  S9 nU R                  UR                  5       5      nSSS5        [        WS[        U5      0S9v   My     M     g! , (       d  f       N.= f7f)zA lazy loader for Documents.)r    r!   Nsource)page_contentmetadata)r#   r   rglobis_diropenr    r!   _clean_datareadr   str)r&   file_patternpftexts        r+   	lazy_loadReadTheDocsLoader.lazy_loadM   s      MML^^)),788::!mmDKKHA++AFFH5D IDHc!f;MNN 8 * IHs   A#B>% B-(B>-
B;7B>c                   SSK Jn  U" US40 U R                  D6nSSS04SSS04/nU R                  b  UR	                  U R                  5        S nUS S S	2    H  u  pgUR                  Xg5      nUc  M    O   Ub%  [        U5      U R                  ::  a  [        U5      nOS
nSR                  UR                  S5       V	s/ sH  o(       d  M  U	PM     sn	5      $ s  sn	f )Nr   r   r   divrolemainidzmain-content 
)r   r   r$   r"   appendfind_get_link_ratior%   _get_clean_textjoinsplit)
r&   datar   soup	html_tagselementtagattrsr;   ts
             r+   r5   ReadTheDocsLoader._clean_dataW   s    %T=CDNNC VV$%dN+,
	
 +T112 $DbD/JCii+G"	 * ?7#;t?W?W#W"7+DDyyTZZ%5;%5!%5;<<;s    	CC)r$   r"   r    r!   r%   r   r#   )NNN)z*.htmz*.htmlg      ?)r'   zUnion[str, Path]r    Optional[str]r!   rT   r"   zOptional[Tuple[str, dict]]r#   zSequence[str]r%   floatr(   zOptional[Any])returnzIterator[Document])rL   r7   rV   r7   )	__name__
__module____qualname____firstlineno____doc__r,   r<   r5   __static_attributes__     r+   r   r      sk    5
 #' $6:"5%(9797  97 	97
 497  97 #97  97vO=r^   r   c                J    / SQn/ SQn[        XU5      nUR                  5       $ )zMReturns cleaned text with newlines preserved and irrelevant elements removed.)scriptnoscriptcanvasmetasvgmapareaaudior/   trackvideoembedobjectparampictureiframeframeframesetnoframesappletformbuttonselectbasestyleimg)r9   r?   ulollih1h2h3h4h5h6pretabletr)_process_elementstrip)rO   elements_to_skipnewline_elementsr;   s       r+   rI   rI   v   s-    :" G7GHD::<r^   c                    U R                  S5      nSR                  S U R                   5       5      n[        U5      S:X  a  gSR                  S U 5       5      n[        U5      [        U5      -  $ )NarD   c              3  6   #    U H  n[        U5      v   M     g 7fN)r7   ).0ss     r+   	<genexpr>"_get_link_ratio.<locals>.<genexpr>   s     B)AAQ)As   r   c              3     #    U HD  nUR                    H1  nU(       d  M  [        UR                  R                  5       5      v   M3     MF     g 7fr   )stringsr7   stringr   )r   linkr   s      r+   r   r      sB      DllF 	#FMM!""" 	#s
   A.A)find_allrJ   stripped_stringslen)sectionlinks
total_text	link_texts       r+   rH   rH      sk    S!EB)A)ABBJ
:!  I y>C
O++r^   c                \  ^^^^^ SSK Jm  SSKJmJm  [        U SS5      n[        U T5      (       d  UT;   a  g[        U T5      (       a  U $ US:X  a  gUT;   a,  SR                  UUUUU4S	 jU R                   5       5      S-   $ SR                  UUUUU4S
 jU R                   5       5      $ )ze
Traverse through HTML tree recursively to preserve newline and skip
unwanted (code/binary) elements
r   r   r   nameNrD   brrE   c              3  h   >#    U H(  n[        UTTT45      (       d  M  [        UTT5      v   M*     g 7fr   
isinstancer   r   childr   r   r   r   r   s     r+   r   #_process_element.<locals>.<genexpr>   s;      -Eec?G%DE L (8:JKK-   22c              3  h   >#    U H(  n[        UTTT45      (       d  M  [        UTT5      v   M*     g 7fr   r   r   s     r+   r   r      s;      
)%#!@A HU$46FGG)r   )	r   r   bs4.elementr   r   getattrr   rJ   children)rO   r   r   tag_namer   r   r   s    `` @@@r+   r   r      s     $(w-H'7##x3C'C	G_	-	-	T		%	%GG  $-- 
 	
 ww 
 
 ))
 
 	
r^   N)rO   r   rV   r7   )r   r   rV   rU   )rO   z$Union[Tag, NavigableString, Comment]r   	List[str]r   r   rV   r7   )
__future__r   pathlibr   typingr   r   r   r   r	   r
   r   r   langchain_core.documentsr   )langchain_community.document_loaders.baser   r   r   r   r   r   r   rI   rH   r   r]   r^   r+   <module>r      sf    "  W W W - @#(d=
 d=N1h,!
1!
!
  !
 		!
r^   