
    dh                     v    S SK r S SKJrJr  S SKrS SKJr  S SKJr  \ R                  " \
5      r " S S\5      rg)    N)AnyList)Document)
BaseLoaderc                   r    \ rS rSrSr SS\S\\   S\4S jjrS\S\4S	 jr	S
\S\
4S jrS\\   4S jrSrg)DiffbotLoader   zLoad `Diffbot` json file.	api_tokenurlscontinue_on_failurec                 (    Xl         X l        X0l        g)zInitialize with API token, ids, and key.

Args:
    api_token: Diffbot API token.
    urls: List of URLs to load.
    continue_on_failure: Whether to continue loading other URLs if one fails.
       Defaults to True.
N)r
   r   r   )selfr
   r   r   s       d/var/www/html/shao/venv/lib/python3.13/site-packages/langchain_community/document_loaders/diffbot.py__init__DiffbotLoader.__init__   s     #	#6     diffbot_apireturnc                     SU 3$ )Nzhttps://api.diffbot.com/v3/ )r   r   s     r   _diffbot_api_urlDiffbotLoader._diffbot_api_url   s    ,[M::r   urlc                     U R                  S5      nU R                  US.n[        R                  " X#SS9nUR                  (       a  UR                  5       $ 0 $ )z'Get Diffbot file from Diffbot REST API.article)tokenr   
   )paramstimeout)r   r
   requestsgetokjson)r   r   diffbot_urlr   responses        r   _get_diffbot_dataDiffbotLoader._get_diffbot_data!   sP     ++I6^^
 <<BG #+++x}}525r   c                 N   [        5       nU R                   HD  n U R                  U5      nSU;   a  US   S   S   OSnSU0nUR                  [	        XES95        MF     U$ ! [
         a9  nU R                  (       a!  [        R                  SU SU 35         S	nAM  UeS	nAff = f)
z>Extract text from Diffbot on all the URLs and return Documentsobjectsr   text source)page_contentmetadatazError fetching or processing z, exception: N)	listr   r&   appendr   	Exceptionr   loggererror)r   docsr   datar*   r.   es          r   loadDiffbotLoader.load.   s    #v99C	--c25>$5FtIq)&1B$c?H$JK    ++LL#@]STRU!VWWG	s   A A!!
B$+,BBB$)r
   r   r   N)T)__name__
__module____qualname____firstlineno____doc__strr   boolr   r   r   r&   r   r7   __static_attributes__r   r   r   r   r      sc    # LP77$(I7DH7;C ;C ;6S 6S 6d8n r   r   )loggingtypingr   r   r    langchain_core.documentsr   )langchain_community.document_loaders.baser   	getLoggerr9   r2   r   r   r   r   <module>rF      s0       - @			8	$1J 1r   