
    dh	                         S r SSKrSSKJrJrJr  SSKJr  SSKJ	r	  SSK
Jr  \R                  " \5      r " S S\5      rg)	zScrapingAnt Web Extractor.    N)IteratorListOptional)
BaseLoader)Document)get_from_envc                   h    \ rS rSrSrSSSS.S\\   S\\   S\\   S	\	S
S4
S jjr
S
\\   4S jrSrg)ScrapingAntLoader   zTurn an url to LLM accessible markdown with `ScrapingAnt`.

For further details, visit: https://docs.scrapingant.com/python-client
NT)api_keyscrape_configcontinue_on_failureurlsr   r   r   returnc                     SSK Jn  U(       d  [        S5      eU=(       d    [	        SS5      nU" US9U l        Xl        X0l        X@l        g! [         a    [        S5      ef = f)	a<  Initialize client.

Args:
    urls: List of urls to scrape.
    api_key: The ScrapingAnt API key. If not specified must have env var
        SCRAPINGANT_API_KEY set.
    scrape_config: The scraping config from ScrapingAntClient.markdown_request
    continue_on_failure: Whether to continue if scraping an url fails.
r   )ScrapingAntClientzL`scrapingant-client` package not found, run `pip install scrapingant-client`zURLs must be provided.r   SCRAPINGANT_API_KEY)tokenN)	scrapingant_clientr   ImportError
ValueErrorr   clientr   r   r   )selfr   r   r   r   r   s         h/var/www/html/shao/venv/lib/python3.13/site-packages/langchain_community/document_loaders/scrapingant.py__init__ScrapingAntLoader.__init__   sm    "	< 566K\)5JK'g6	*#6   	8 	s   A A#c              #   v  #    U R                   b  U R                   O0 nU R                   HD  n U R                  R                  " SSU0UD6n[	        UR
                  SUR                  0S9v   MF     g! [         a9  nU R                  (       a!  [        R                  SU SU 35         SnAM  UeSnAff = f7f)zFetch data from ScrapingAnt.Nurl)page_contentmetadatazError fetching data from z, exception:  )r   r   r   markdown_requestr   markdownr   	Exceptionr   loggererror)r   r   r   resultes        r   	lazy_loadScrapingAntLoader.lazy_load3   s      /3.@.@.L**RT99C
55O#OO!'#VZZ0    ++LL#<SEqc!RSSG	s5   +B9A A3.B93
B6=,B1)B9/B11B66B9)r   r   r   r   )__name__
__module____qualname____firstlineno____doc__r   strr   dictboolr   r   r   r)   __static_attributes__r!       r   r
   r
      se     "&(,$(73i7 #	7
  ~7 "7 
7@8H- r4   r
   )r/   loggingtypingr   r   r   langchain_core.document_loadersr   langchain_core.documentsr   langchain_core.utilsr   	getLogger__file__r%   r
   r!   r4   r   <module>r<      s6       + + 6 - -			8	$5
 5r4   