
    dh)                     R    S SK JrJrJr  S SKJr  S SKJr  S SKJ	r	   " S S\5      r
g)    )IteratorLiteralOptional)
BaseLoader)Document)get_from_envc            
       d    \ rS rSrSrSSSS.S\S\\   S\S	   S
\\   4S jjr	S\
\   4S jrSrg)SpiderLoader   zLoad web pages as Documents using Spider AI.

Must have the Python package `spider-client` installed and a Spider API key.
See https://spider.cloud for more.
Nscrape)api_keymodeparamsurlr   r   r   crawlr   c                    Uc  SSS.n SSK Jn  US;  a  [        S	U S
35      eU=(       d    [	        SS5      nU" US9U l         Xl        X0l        X@l        g! [         a    [        S5      ef = f)a  Initialize with API key and URL.

Args:
    url: The URL to be processed.
    api_key: The Spider API key. If not specified, will be read from env
    var `SPIDER_API_KEY`.
    mode: The mode to run the loader in. Default is "scrape".
         Options include "scrape" (single page) and "crawl" (with deeper
         crawling following subpages).
    params: Additional parameters for the Spider API.
NmarkdownT)return_formatmetadatar   )SpiderzB`spider` package not found, please run `pip install spider-client`r   zUnrecognized mode 'z%'. Expected one of 'scrape', 'crawl'.r   SPIDER_API_KEY)r   )spiderr   ImportError
ValueErrorr   r   r   r   )selfr   r   r   r   r   s         c/var/www/html/shao/venv/lib/python3.13/site-packages/langchain_community/document_loaders/spider.py__init__SpiderLoader.__init__   s    & >!+ F
	%
 **%dV+PQ 
 F\)5EFW-	  	T 	s   A A.returnc              #     #    / nU R                   S:X  aG  U R                  R                  U R                  U R                  S9nU(       a  UR                  U5        OVU R                   S:X  aF  U R                  R                  U R                  U R                  S9nU(       a  UR                  U5        U H  nU R                   S:X  a8  US   R                  SS5      nUS   R                  S0 5      nUb  [        XES	9v   U R                   S:X  d  M]  UR                  SS5      nUR                  S0 5      nUc  M  [        UUS	9v   M     g7f)
z+Load documents based on the specified mode.r   )r   r   r   content r   N)page_contentr   )
r   r   
scrape_urlr   r   append	crawl_urlextendgetr   )r   spider_docsresponsedocr$   r   s         r   	lazy_loadSpiderLoader.lazy_load:   s    99 {{--dhht{{-KH""8,YY'!{{,,TXXdkk,JH""8,CyyH$"1vzz)R8 q6::j"5+"PPyyG#"wwy"5 77:r2+"%1!) % s   DE%E:E)r   r   r   r   )__name__
__module____qualname____firstlineno____doc__strr   r   dictr   r   r   r-   __static_attributes__     r   r
   r
      s]     "&+3!%)) #	)
 '() )V$8H- $r8   r
   N)typingr   r   r   langchain_core.document_loadersr   langchain_core.documentsr   langchain_core.utilsr   r
   r7   r8   r   <module>r=      s"    . . 6 - -V: Vr8   