§
    šŠtj‚D  ã                   ón   — d dl Z d dlZd dlmZmZmZmZ d dlmZ d dl	m
Z
 d dlmZ  G d„ de¦  «        ZdS )é    N)ÚAnyÚIteratorÚLiteralÚOptional)Ú
BaseLoader)ÚDocument)Úget_from_envc                   ó¤   — e Zd ZdZ	 dddddddœdee         dee         dee         dee         d	ed
         dee         fd„Zde	e
         fd„ZdS )ÚFireCrawlLoadera!
  
    FireCrawlLoader document loader integration

    Setup:
        Install ``firecrawl-py``,``langchain_community`` and set environment variable ``FIRECRAWL_API_KEY``.

        .. code-block:: bash

            pip install -U firecrawl-py langchain_community
            export FIRECRAWL_API_KEY="your-api-key"

    Instantiate:
        .. code-block:: python

            from langchain_community.document_loaders import FireCrawlLoader

            loader = FireCrawlLoader(
                url = "https://firecrawl.dev",
                mode = "crawl"
                # other params = ...
            )

    Lazy load:
        .. code-block:: python

            docs = []
            docs_lazy = loader.lazy_load()

            # async variant:
            # docs_lazy = await loader.alazy_load()

            for doc in docs_lazy:
                docs.append(doc)
            print(docs[0].page_content[:100])
            print(docs[0].metadata)

        .. code-block:: python

            Introducing [Smart Crawl!](https://www.firecrawl.dev/smart-crawl)
             Join the waitlist to turn any web
            {'ogUrl': 'https://www.firecrawl.dev/', 'title': 'Home - Firecrawl', 'robots': 'follow, index', 'ogImage': 'https://www.firecrawl.dev/og.png?123', 'ogTitle': 'Firecrawl', 'sitemap': {'lastmod': '2024-08-12T00:28:16.681Z', 'changefreq': 'weekly'}, 'keywords': 'Firecrawl,Markdown,Data,Mendable,Langchain', 'sourceURL': 'https://www.firecrawl.dev/', 'ogSiteName': 'Firecrawl', 'description': 'Firecrawl crawls and converts any website into clean markdown.', 'ogDescription': 'Turn any website into LLM-ready data.', 'pageStatusCode': 200, 'ogLocaleAlternate': []}

    Async load:
        .. code-block:: python

            docs = await loader.aload()
            print(docs[0].page_content[:100])
            print(docs[0].metadata)

        .. code-block:: python

            Introducing [Smart Crawl!](https://www.firecrawl.dev/smart-crawl)
             Join the waitlist to turn any web
            {'ogUrl': 'https://www.firecrawl.dev/', 'title': 'Home - Firecrawl', 'robots': 'follow, index', 'ogImage': 'https://www.firecrawl.dev/og.png?123', 'ogTitle': 'Firecrawl', 'sitemap': {'lastmod': '2024-08-12T00:28:16.681Z', 'changefreq': 'weekly'}, 'keywords': 'Firecrawl,Markdown,Data,Mendable,Langchain', 'sourceURL': 'https://www.firecrawl.dev/', 'ogSiteName': 'Firecrawl', 'description': 'Firecrawl crawls and converts any website into clean markdown.', 'ogDescription': 'Turn any website into LLM-ready data.', 'pageStatusCode': 200, 'ogLocaleAlternate': []}

    NÚcrawl)ÚqueryÚapi_keyÚapi_urlÚmodeÚparamsÚurlr   r   r   r   )r   ÚscrapeÚmapÚextractÚsearchr   c                óÊ  — 	 ddl m} n# t          $ r t          d¦  «        ‚w xY w|dvrt          d|› d�¦  «        ‚|dv r|st          d¦  «        ‚|d	k    r(|s&|r|                     d
¦  «        st          d¦  «        ‚|pt          dd¦  «        }|pt          j        d¦  «        pd} |||¬¦  «        | _         |pd| _        || _	        |pi | _
        |�|| j
        d
<   dS dS )aÑ  Initialize with API key and url.

        Args:
            url: The url to be crawled.
            api_key: The Firecrawl API key. If not specified will be read from env var
                FIRECRAWL_API_KEY. Get an API key
            api_url: The Firecrawl API URL. If not specified will be read from env var
                FIRECRAWL_API_URL or defaults to https://api.firecrawl.dev.
            mode: The mode to run the loader in. Default is "crawl".
                 Options include "scrape" (single url),
                 "crawl" (all accessible sub pages),
                 "map" (returns list of links that are semantically related).
                 "extract" (extracts structured data from a page).
                 "search" (search for data across the web).
            params: The parameters to pass to the Firecrawl API.
                Examples include crawlerOptions.
                For more details, visit: https://github.com/mendableai/firecrawl-py
        r   )ÚFirecrawlAppzD`firecrawl` package not found, please run `pip install firecrawl-py`)r   r   r   r   r   r   úInvalid mode 'zT'.
                Allowed: 'crawl', 'scrape', 'search', 'map', 'extract', 'search'.)r   r   r   r   z2Url must be provided for modes other than 'search'r   r   z&Query must be provided for search moder   ÚFIRECRAWL_API_KEYÚFIRECRAWL_API_URLzhttps://api.firecrawl.dev)r   r   Ú N)Ú	firecrawlr   ÚImportErrorÚ
ValueErrorÚgetr	   ÚosÚgetenvr   r   r   )	Úselfr   r   r   r   r   r   r   Úresolved_api_urls	            úl/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langchain_community/document_loaders/firecrawl.pyÚ__init__zFireCrawlLoader.__init__F   sh  € ð:	Ø.Ð.Ð.Ð.Ð.Ð.Ð.øÝð 	ð 	ð 	ÝØVñô ð ð	øøøð ÐPÐPÐPÝðU 4ð Uð Uð Uñô ð ð
 Ð8Ð8Ð8ÀÐ8ÝÐQÑRÔRÐRØ�8ÒÐ UÐ¨vÐ¸&¿*º*ÀWÑ:MÔ:MÐÝÐEÑFÔFÐFàÐI�\¨)Ð5HÑIÔIˆð ÐT•r”yÐ!4Ñ5Ô5ÐTÐ9Tð 	ð &˜¨gÐ?OÐPÑPÔPˆŒØ�9˜"ˆŒØˆŒ	Ø�l ˆŒØÐØ#(ˆDŒK˜Ñ Ð Ð ð Ðs   ‚	 ‰#Úreturnc           
   #   ó@  ‡K  — g }| j         dk    rIh d£Šˆfd„| j                             ¦   «         D ¦   «         }d|d<    | j        j        | j        fi |¤Žg}�n$| j         dk    rè| j        st          d¦  «        ‚h d£Šˆfd	„| j                             ¦   «         D ¦   «         }d|d<    | j        j        | j        fi |¤Ž}t          |t          ¦  «        r>| 
                    d
g ¦  «        }t          |t          ¦  «        rt          |¦  «        ng }�njt          |d
g ¦  «        }t          |t          ¦  «        rt          |¦  «        ng }�n1| j         dk    �r | j        st          d¦  «        ‚h d£Šˆfd„| j                             ¦   «         D ¦   «         }d|d<    | j        j        | j        fi |¤Ž}t          |t          ¦  «        r=| 
                    d¦  «        }t          |t          ¦  «        rt          |¦  «        ng }�nwt          |d¦  «        r8t          |d¦  «        }t          |t          ¦  «        rt          |¦  «        ng }�n/t          |t          ¦  «        }|rt          |¦  «        ng }�n| j         dk    rm| j        st          d¦  «        ‚h d£Šˆfd„| j                             ¦   «         D ¦   «         }d|d<   t           | j        j        | j        gfi |¤Ž¦  «        g}�n�| j         dk    �rih d£Šˆfd„| j                             ¦   «         D ¦   «         }d|d<    | j        j        d3d| j         
                    d¦  «        i|¤Ž}t          |t          ¦  «        rt          |¦  «        }�ng }	g }
t          |t          ¦  «        r?| 
                    d¦  «        | 
                    d¦  «        | 
                    d¦  «        g}
n2t          |dd ¦  «        t          |dd ¦  «        t          |dd ¦  «        g}
d|
d         fd|
d         fd|
d         ffD �]=\  }}|sŒ	|D �]0}t          |t          ¦  «        st          |dd ¦  «        n| 
                    d¦  «        }t          |t          ¦  «        st          |dd ¦  «        n| 
                    d¦  «        }t          |t          ¦  «        st          |d d ¦  «        n| 
                    d ¦  «        }|p|p|pd!}d"„ ||t          |t          ¦  «        st          |d#d ¦  «        n| 
                    d#¦  «        |d$œ                     ¦   «         D ¦   «         }|	                     ||d%œ¦  «         �Œ2�Œ?|	}nt          d&| j         › d'�¦  «        ‚|D �]}| j         dk    �r~t          |t          ¦  «        r|}i }�nÛt          |t          ¦  «        r�| 
                    d¦  «        p| 
                    d(¦  «        pd!}t          |t          ¦  «        r|nt          |pd!¦  «        }d)„ | 
                    d¦  «        | 
                    d ¦  «        d*œ                     ¦   «         D ¦   «         }�n)t          |d¦  «        st          |d¦  «        r~t          |dd!¦  «        pt          |d(d!¦  «        }t          |t          ¦  «        r|nt          |pd!¦  «        }i }t          |dd ¦  «        }t          |d d ¦  «        }|�||d<   |�||d <   �n‹t          |¦  «        }i }�nx| j         dk    rt          |¦  «        }i }�nZ| j         dk    r˜t          |t          ¦  «        rp| 
                    d+¦  «        pd!}t          |t          ¦  «        r|nt          |pd!¦  «        }| 
                    d,i ¦  «        }t          |t          ¦  «        r|ni }�nÊt          |¦  «        }i }�n·t          |t          ¦  «        r| 
                    d+¦  «        p*| 
                    d-¦  «        p| 
                    d.d!¦  «        }t          |t          ¦  «        r|nt          |pd!¦  «        }| 
                    d,i ¦  «        }nnt          |d+d ¦  «        p!t          |d-d ¦  «        pt          |d.d!¦  «        }t          |t          ¦  «        r|nt          |pd!¦  «        }t          |d,i ¦  «        pi }t          |t          ¦  «        s t          |d/¦  «        r)t'          |j        ¦  «        r|                     ¦   «         }ngt+          j        |¦  «        rt+          j        |¦  «        }n>t          |d0¦  «        rt          t1          |¦  «        ¦  «        }nd1t          |¦  «        i}|s�Œ	t3          ||¬2¦  «        V — �Œd S )4Nr   >   ÚproxyÚmobileÚactionsÚformatsÚheadersÚmax_ageÚparsersÚtimeoutÚlocationÚuse_mockÚwait_forÚ	block_adsÚ	fast_modeÚexclude_tagsÚinclude_tagsÚstore_in_cacheÚonly_main_contentÚremove_base64_imagesÚskip_tls_verificationc                 ó$   •— i | ]\  }}|‰v ¯	||“ŒS © r=   ©Ú.0ÚkÚvÚalloweds      €r%   ú
<dictcomp>z-FireCrawlLoader.lazy_load.<locals>.<dictcomp>š   ó$   ø€ ÐKÐKÐK™t˜q !¸aÀ7¸l¸l�a˜¸l¸l¸ló    Ú	langchainÚintegrationr   zURL is required for crawl mode>   ÚdelayÚlimitÚpromptr0   ÚwebhookÚexclude_pathsÚinclude_pathsÚpoll_intervalÚignore_sitemapÚscrape_optionsÚmax_concurrencyÚallow_subdomainsÚcrawl_entire_domainÚmax_discovery_depthÚzero_data_retentionÚallow_external_linksÚignore_query_parametersc                 ó$   •— i | ]\  }}|‰v ¯	||“ŒS r=   r=   r>   s      €r%   rC   z-FireCrawlLoader.lazy_load.<locals>.<dictcomp>³   rD   rE   Údatar   zURL is required for map mode>   rI   r   Úsitemapr0   r1   Úinclude_subdomainsc                 ó$   •— i | ]\  }}|‰v ¯	||“ŒS r=   r=   r>   s      €r%   rC   z-FireCrawlLoader.lazy_load.<locals>.<dictcomp>È   rD   rE   Úlinksr   z URL is required for extract mode>   ÚagentrJ   Úschemar0   Úshow_sourcesrN   Úsystem_promptrP   Úenable_web_searchÚignore_invalid_urlsrV   c                 ó$   •— i | ]\  }}|‰v ¯	||“ŒS r=   r=   r>   s      €r%   rC   z-FireCrawlLoader.lazy_load.<locals>.<dictcomp>æ   rD   rE   r   >   ÚtbsrI   Úsourcesr0   r1   Ú
categoriesrP   rc   c                 ó$   •— i | ]\  }}|‰v ¯	||“ŒS r=   r=   r>   s      €r%   rC   z-FireCrawlLoader.lazy_load.<locals>.<dictcomp>ô   rD   rE   r   ÚwebÚnewsÚimagesr   é   é   r   ÚtitleÚdescriptionr   c                 ó   — i | ]
\  }}|®||“ŒS ©Nr=   ©r?   r@   rA   s      r%   rC   z-FireCrawlLoader.lazy_load.<locals>.<dictcomp>$  s/   € ð (ð (ð (á $  1ð  !˜}ð ˜qð  -˜}˜}rE   Úcategory)r   rn   rs   Útype)ÚmarkdownÚmetadatar   zJ'.
                Allowed: 'crawl', 'scrape', 'map', 'extract', 'search'.Úhrefc                 ó   — i | ]
\  }}|®||“ŒS rq   r=   rr   s      r%   rC   z-FireCrawlLoader.lazy_load.<locals>.<dictcomp>F  s/   € ð ð ð á ˜A˜qð ˜=ð ˜1ð
 )˜=˜=rE   )rn   ro   ru   rv   ÚhtmlÚrawHtmlÚ
model_dumpÚ__dict__Úvalue)Úpage_contentrv   r=   )r   r   Úitemsr   r   r   r   r   Ú
isinstanceÚdictr    ÚlistÚgetattrr   ÚhasattrÚstrr   r   ÚappendÚcallabler{   ÚdataclassesÚis_dataclassÚasdictÚvarsr   )r#   Úfirecrawl_docsÚkwargsÚcrawl_responserY   Úmap_responser]   Úis_listÚsearch_dataÚresultsÚ
containersÚkindr   ÚitemÚurl_valÚ	title_valÚdesc_valÚcontent_valÚmetadata_valÚdocr~   ÚmetaÚpage_content_valuern   ro   Úmarkdown_valueÚmetadata_objÚcontent_valuerB   s                               @r%   Ú	lazy_loadzFireCrawlLoader.lazy_load�   s  øè è € à$&ˆØŒ9˜Ò Ð ðð ð ˆGð* LÐKÐKÐK t¤{×'8Ò'8Ñ':Ô':ÐKÑKÔKˆFØ$/ˆF�=Ñ!Ø3˜dœnÔ3°D´HÐGÐGÀÐGÐGÐHˆN‰NØŒY˜'Ò!Ð!Ø”8ð CÝ Ð!AÑBÔBÐBðð ð ˆGð& LÐKÐKÐK t¤{×'8Ò'8Ñ':Ô':ÐKÑKÔKˆFØ$/ˆF�=Ñ!Ø1˜Tœ^Ô1°$´(ÐEÐE¸fÐEÐEˆNå˜.­$Ñ/Ô/ð NØ%×)Ò)¨&°"Ñ5Ô5�Ý/9¸$ÅÑ/EÔ/EÐ!M¥ d¡¤ È2�‘å˜~¨v°rÑ:Ô:�Ý/9¸$ÅÑ/EÔ/EÐ!M¥ d¡¤ È2�‘ØŒY˜%ÒÑØ”8ð AÝ Ð!?Ñ@Ô@Ð@ðð ð ˆGð LÐKÐKÐK t¤{×'8Ò'8Ñ':Ô':ÐKÑKÔKˆFØ$/ˆF�=Ñ!Ø-˜4œ>Ô-¨d¬hÐAÐA¸&ÐAÐAˆLõ ˜,­Ñ-Ô-ð GØ$×(Ò(¨Ñ1Ô1�Ý0:¸5Å$Ñ0GÔ0GÐ!O¥ e¡¤ ÈR�‘Ý˜ wÑ/Ô/ð GÝ ¨gÑ6Ô6�Ý0:¸5Å$Ñ0GÔ0GÐ!O¥ e¡¤ ÈR�‘å$ \µ4Ñ8Ô8�Ø7>Ð!F¥ lÑ!3Ô!3Ð!3ÀB�‘ØŒY˜)Ò#Ð#Ø”8ð EÝ Ð!CÑDÔDÐDðð ð ˆGð LÐKÐKÐK t¤{×'8Ò'8Ñ':Ô':ÐKÑKÔKˆFØ$/ˆF�=Ñ!Ý!Ð"8 $¤.Ô"8¸$¼(¸Ð"NÐ"NÀvÐ"NÐ"NÑOÔOÐPˆN‰NØŒY˜(Ò"Ñ"ð	ð 	ð 	ˆGð LÐKÐKÐK t¤{×'8Ò'8Ñ':Ô':ÐKÑKÔKˆFØ$/ˆF�=Ñ!Ø/˜$œ.Ô/ð ð Ø”k—o’o gÑ.Ô.ðØ28ðð ˆKõ ˜+¥tÑ,Ô,ð 9)Ý!% kÑ!2Ô!2�‘ð 13�Ø�
Ý˜k­4Ñ0Ô0ð à#Ÿš¨Ñ.Ô.Ø#Ÿš¨Ñ/Ô/Ø#Ÿš¨Ñ1Ô1ð"�J�Jõ   ¨U°DÑ9Ô9Ý ¨V°TÑ:Ô:Ý ¨X°tÑ<Ô<ð"�Jð ˜J qœMÐ*Ø˜Z¨œ]Ð+Ø˜z¨!œ}Ð-ð$ð &ñ &‘K�D˜%ð
 !ð !Ø Ø %ð ñ ˜õ $.¨dµDÑ#9Ô#9ð1�G D¨%°Ñ6Ô6Ð6à!%§¢¨%¡¤ð  õ $.¨dµDÑ#9Ô#9ð3�G D¨'°4Ñ8Ô8Ð8à!%§¢¨'Ñ!2Ô!2ð "õ $.¨dµDÑ#9Ô#9ð9�G D¨-¸Ñ>Ô>Ð>à!%§¢¨-Ñ!8Ô!8ð !ð
 '/Ð&L°)Ð&L¸wÐ&LÈ"˜ð(ð (ð (/Ø)2å'1°$½Ñ'=Ô'=ð-:­G°D¸*ÀdÑ,KÔ,KÐ,Kà%)§X¢X¨jÑ%9Ô%9Ø(,ð)ð )÷ $še™gœgð(ñ (ô (˜ð  ŸšØ)4À,ÐOÐOñô ð ñ ñ;ð@ ")��åðK 4¤9ð Kð Kð Kñô ð ð "ð `	ñ `	ˆCØŒy˜EÒ!Ñ!å˜c¥3Ñ'Ô'ð $Ø(+�LØ+-�D‘DÝ ¥TÑ*Ô*ð !Ø),¯ª°©¬Ð)P¸3¿7º7À6¹?¼?Ð)PÈbÐ&õ &Ð&8½#Ñ>Ô>ð;Ð*Ð*å Ð!3Ð!9°rÑ:Ô:ð !ð
ð ð &)§W¢W¨WÑ%5Ô%5Ø+.¯7ª7°=Ñ+AÔ+Að%ð %÷  š%™'œ'ðñ ô �D‘Dõ ˜S %Ñ(Ô(ð ­G°C¸Ñ,AÔ,Að Ý)0°°e¸RÑ)@Ô)@ð *ÅGØ˜V RñEô EÐ&õ
 &Ð&8½#Ñ>Ô>ð;Ð*Ð*å Ð!3Ð!9°rÑ:Ô:ð !ð
 �DÝ# C¨°$Ñ7Ô7�EÝ")¨#¨}¸dÑ"CÔ"C�KØÐ(Ø(-˜˜W™Ø"Ð.Ø.9˜˜]Ñ+ùå#& s¡8¤8�LØ�D‘DØ”˜iÒ'Ð'Ý" 3™xœx�Ø�‘Ø”˜hÒ&Ð&å˜c¥4Ñ(Ô(ð Ø%(§W¢W¨ZÑ%8Ô%8Ð%>¸B�Nõ & nµcÑ:Ô:ð7˜˜å  Ð!5°2Ñ6Ô6ð !ð
 $'§7¢7¨:°rÑ#:Ô#:�LÝ+5°lÅDÑ+IÔ+IÐQ˜<˜<Èr�D‘Då#& s¡8¤8�LØ�D‘Då˜c¥4Ñ(Ô(ð >àŸš 
Ñ+Ô+ÐX¨s¯wªw°v©¬ÐXÀ#Ç'Â'È)ÐUWÑBXÔBXð "õ
 & mµSÑ9Ô9ð6˜˜å  Ð!4°"Ñ5Ô5ð !ð
 Ÿ7š7 :¨rÑ2Ô2�D�Dõ    Z°Ñ6Ô6ð 7Ý" 3¨°Ñ5Ô5ð7å" 3¨	°2Ñ6Ô6ð "õ & mµSÑ9Ô9ð6˜˜å  Ð!4°"Ñ5Ô5ð !õ
 # 3¨
°BÑ7Ô7Ð=¸2�Dõ " $­Ñ-Ô-ð 4Ý˜t \Ñ2Ô2ð 4µxÀÄÑ7PÔ7Pð 4Ø#ŸšÑ0Ô0˜˜Ý$Ô1°$Ñ7Ô7ð 4Ý*Ô1°$Ñ7Ô7˜˜Ý   zÑ2Ô2ð 4Ý#¥D¨¡J¤JÑ/Ô/˜˜à '­¨T©¬Ð3˜Øð ÙÝØ)Øðñ ô ð ð ð ñ ð{`	ð `	rE   rq   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r…   r   r�   r&   r   r   r¡   r=   rE   r%   r   r   
   sÔ   € € € € € ð7ð 7ðz "ð9)ð  $Ø!%Ø!%ØGNØ!%ð9)ð 9)ð 9)à�cŒ]ð9)ð ˜Œ}ð	9)ð
 ˜#”ð9)ð ˜#”ð9)ð ÐCÔDð9)ð ˜”ð9)ð 9)ð 9)ð 9)ðvX˜8 HÔ-ð Xð Xð Xð Xð Xð XrE   r   )rˆ   r!   Útypingr   r   r   r   Úlangchain_core.document_loadersr   Úlangchain_core.documentsr   Úlangchain_core.utilsr	   r   r=   rE   r%   ú<module>rª      s²   ðØ Ð Ð Ð Ø 	€	€	€	Ø 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3à 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø -Ð -Ð -Ð -Ð -Ð -Ø -Ð -Ð -Ð -Ð -Ð -ðOð Oð Oð Oð O�jñ Oô Oð Oð Oð OrE   