
    dh	                         S r SSKrSSKJrJrJrJr  SSKJr  SSK	J
r
  SSKJr  \R                  " \5      r " S S\5      rg)	zScrapfly Web Reader.    N)IteratorListLiteralOptional)
BaseLoader)Document)get_from_envc                   t    \ rS rSrSrSSSSS.S\\   S\\   S	\S
   S\\	   S\
SS4S jjrS\\   4S jrSrg)ScrapflyLoader   z{Turn a url to llm accessible markdown with `Scrapfly.io`.

For further details, visit: https://scrapfly.io/docs/sdk/python
NmarkdownT)api_keyscrape_formatscrape_configcontinue_on_failureurlsr   r   )r   textr   r   returnc                     SSK Jn  U(       d  [        S5      eU=(       d    [	        SS5      nU" US9U l         Xl        X0l        X@l        XPl        g! [         a    [        S5      ef = f)	al  Initialize client.

Args:
    urls: List of urls to scrape.
    api_key: The Scrapfly API key. If not specified must have env var
        SCRAPFLY_API_KEY set.
    scrape_format: Scrape result format, one or "markdown" or "text".
    scrape_config: Dictionary of ScrapFly scrape config object.
    continue_on_failure: Whether to continue if scraping a url fails.
r   )ScrapflyClientzC`scrapfly` package not found, please run `pip install scrapfly-sdk`zURLs must be provided.r   SCRAPFLY_API_KEY)keyN)	scrapflyr   ImportError
ValueErrorr	   r   r   r   r   )selfr   r   r   r   r   r   s          e/var/www/html/shao/venv/lib/python3.13/site-packages/langchain_community/document_loaders/scrapfly.py__init__ScrapflyLoader.__init__   sq    &	/
 566H\)5GH&73	**#6   	U 	s   A A)c              #     #    SSK Jn  U R                  b  U R                  O0 nU R                   HM  n U R                   R	                  U" U4SU R
                  0UD65      n[        UR                  S   SU0S9v   MO     g ! [         a9  nU R                  (       a!  [        R                  SU SU 35         S nAM  UeS nAff = f7f)	Nr   )ScrapeConfigformatcontenturl)page_contentmetadatazError fetching data from z, exception: )r   r!   r   r   scraper   r   scrape_result	Exceptionr   loggererror)r   r!   r   r$   responsees         r   	lazy_loadScrapflyLoader.lazy_load5   s     ).2.@.@.L**RT99C==// QT-?-?Q=Q !)!7!7	!B#S\    ++LL#<SEqc!RSSG	s5   1CA	B=C
C,C 8C>C  CC)r   r   r   r   r   )__name__
__module____qualname____firstlineno____doc__r   strr   r   dictboolr   r   r   r.   __static_attributes__     r   r   r      sx     "&5?(,$( 73i 7 #	 7
 12 7  ~ 7 " 7 
 7D8H- r:   r   )r4   loggingtypingr   r   r   r   langchain_core.document_loadersr   langchain_core.documentsr   langchain_core.utilsr	   	getLogger__file__r*   r   r9   r:   r   <module>rB      s6      4 4 6 - -			8	$9Z 9r:   