§
    šŠtjÑ	  ã                   óŠ   — d Z ddlZddlmZmZmZmZ ddlmZ ddl	m
Z
 ddlmZ  ej        e¦  «        Z G d„ de¦  «        ZdS )	zScrapfly Web Reader.é    N)ÚIteratorÚListÚLiteralÚOptional)Ú
BaseLoader)ÚDocument)Úget_from_envc                   ó†   — e Zd ZdZdddddœdee         dee         ded	         d
ee         de	ddfd„Z
dee         fd„ZdS )ÚScrapflyLoaderzƒTurn a url to llm accessible markdown with `Scrapfly.io`.

    For further details, visit: https://scrapfly.io/docs/sdk/python
    NÚmarkdownT)Úapi_keyÚscrape_formatÚscrape_configÚcontinue_on_failureÚurlsr   r   )r   Útextr   r   Úreturnc                óð   — 	 ddl m} n# t          $ r t          d¦  «        ‚w xY w|st          d¦  «        ‚|pt	          dd¦  «        } ||¬¦  «        | _         || _        || _        || _        || _        dS )	a¬  Initialize client.

        Args:
            urls: List of urls to scrape.
            api_key: The Scrapfly API key. If not specified must have env var
                SCRAPFLY_API_KEY set.
            scrape_format: Scrape result format, one or "markdown" or "text".
            scrape_config: Dictionary of ScrapFly scrape config object.
            continue_on_failure: Whether to continue if scraping a url fails.
        r   )ÚScrapflyClientzC`scrapfly` package not found, please run `pip install scrapfly-sdk`zURLs must be provided.r   ÚSCRAPFLY_API_KEY)ÚkeyN)	Úscrapflyr   ÚImportErrorÚ
ValueErrorr	   r   r   r   r   )Úselfr   r   r   r   r   r   s          úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langchain_community/document_loaders/scrapfly.pyÚ__init__zScrapflyLoader.__init__   s³   € ð&	Ø/Ð/Ð/Ð/Ð/Ð/Ð/øÝð 	ð 	ð 	ÝØUñô ð ð	øøøð ð 	7ÝÐ5Ñ6Ô6Ð6ØÐH�\¨)Ð5GÑHÔHˆØ&˜¨7Ð3Ñ3Ô3ˆŒØˆŒ	Ø*ˆÔØ*ˆÔØ#6ˆÔ Ð Ð s   ‚	 ‰#c              #   ó^  K  — ddl m} | j        �| j        ni }| j        D ]Œ}	 | j                               ||fd| j        i|¤Ž¦  «        }t          |j        d         d|i¬¦  «        V — ŒL# t          $ r4}| j	        r!t                               d|› d|› �¦  «         n|‚Y d }~Œ…d }~ww xY wd S )	Nr   )ÚScrapeConfigÚformatÚcontentÚurl)Úpage_contentÚmetadatazError fetching data from z, exception: )r   r   r   r   Úscraper   r   Úscrape_resultÚ	Exceptionr   ÚloggerÚerror)r   r   r   r"   ÚresponseÚes         r   Ú	lazy_loadzScrapflyLoader.lazy_load5   s%  è è € Ø)Ð)Ð)Ð)Ð)Ð)à.2Ô.@Ð.L˜Ô*Ð*ÐRTˆØ”9ð 	ð 	ˆCðØœ=×/Ò/Ø �L ÐQÐQ¨TÔ-?ÐQÀ=ÐQÐQñô �õ Ø!)Ô!7¸	Ô!BØ# S˜\ðñ ô ð ð ð ð øõ ð ð ð ØÔ+ð Ý—L’LÐ!R¸SÐ!RÐ!RÈqÐ!RÐ!RÑSÔSÐSÐSà�Gð TÐSÐSÐSÐSøøøøðøøøð	ð 	s   £AA,Á,
B*Á6*B%Â%B*)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ústrr   r   ÚdictÚboolr   r   r   r,   © ó    r   r   r      s·   € € € € € ðð ð "&Ø5?Ø(,Ø$(ð 7ð  7ð  7à�3Œið 7ð ˜#”ð	 7ð
 Ð1Ô2ð 7ð   ”~ð 7ð "ð 7ð 
ð 7ð  7ð  7ð  7ðD˜8 HÔ-ð ð ð ð ð ð r5   r   )r0   ÚloggingÚtypingr   r   r   r   Úlangchain_core.document_loadersr   Úlangchain_core.documentsr   Úlangchain_core.utilsr	   Ú	getLoggerÚ__file__r(   r   r4   r5   r   ú<module>r=      s´   ðØ Ð à €€€Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4Ð 4à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø -Ð -Ð -Ð -Ð -Ð -Ø -Ð -Ð -Ð -Ð -Ð -à	ˆÔ	˜8Ñ	$Ô	$€ð9ð 9ð 9ð 9ð 9�Zñ 9ô 9ð 9ð 9ð 9r5   