
    dh                         S SK r S SKrS SKrS SKJrJrJr  S SKJr  S SK	r	S SK
Jr  S SKJr  S SKJr  S SKJr   " S S	5      r " S
 S\5      r " S S\5      rg)    N)AnyListOptional)urljoin)Document)HTTPBasicAuth)
BaseLoader)UnstructuredBaseLoaderc            
       `    \ rS rSrSrS\S\S\4S jrS\S\S	\S
\\   S\	4
S jr
S\4S jrSrg)LakeFSClient   zClient for lakeFS.lakefs_access_keylakefs_secret_keylakefs_endpointc                    SR                  USS/5      U l        [        X5      U l         [        R
                  " [        U R                  S5      U R                  S9nUR                  5         g ! [         a    [        S5      ef = f)N/apizv1/healthcheckauthz<lakeFS server isn't accessible. Make sure lakeFS is running.)
join_LakeFSClient__endpointr   _LakeFSClient__authrequestsgetr   raise_for_status	Exception
ValueError)selfr   r   r   health_checks        c/var/www/html/shao/venv/lib/python3.13/site-packages/langchain_community/document_loaders/lakefs.py__init__LakeFSClient.__init__   s|     ((OUE#BC#$5I	#<<7dkkL ))+ 	N 	s   AA/ /Breporefpathpresignreturnc           	      4   X4S.n[         R                  R                  U5      n[        U R                  SU SU SU 35      n[
        R                  " XpR                  S9nUR                  5         UR                  5       n	[        [        S U	S   5      5      $ )N)prefixr'   zrepositories/z/refs/z/objects/ls?r   c                     U S   U S   4$ )Nr&   physical_address )ress    r!   <lambda>)LakeFSClient.ls_objects.<locals>.<lambda>1   s    S[#.@*AB    results)urllibparse	urlencoder   r   r   r   r   r   jsonlistmap)
r   r$   r%   r&   r'   qpeqpobjects_ls_endpointolsr	olsr_jsons
             r!   
ls_objectsLakeFSClient.ls_objects$   s     1ll$$R(%OO}TF&\#O
 ||/kkBIIK	BIiDX
 	
r1   c                     U R                   S-   n[        R                  " XR                  S9nUR	                  5         UR                  5       nUS   S   $ )Nconfigr   storage_configpre_sign_support)r   r   r   r   r   r6   )r   config_endpointresponserA   s       r!   is_presign_supported!LakeFSClient.is_presign_supported5   sK    //H4<<kkB!!#&'(:;;r1   )__auth
__endpointN)__name__
__module____qualname____firstlineno____doc__strr"   r   boolr   r>   rF   __static_attributes__r-   r1   r!   r   r      sc      	$

!
),
7?~
	
"<d <r1   r   c                       \ rS rSr% Sr\\S'   \\S'   \\S'      SS\S\S	\S\\   S\\   S\\   4S
 jjrS\SS4S jr	S\SS4S jr
S\SS4S jrS\\   4S jrSS jrSrg)LakeFSLoader=   zLoad from `lakeFS`.r$   r%   r&   Nr   r   r   c                     [        XU5      U l        Ub  US:X  a  SO
[        U5      U l        Ub  US:X  a  SO
[        U5      U l        Uc  SU l        g[        U5      U l        g)a  

:param lakefs_access_key: [required] lakeFS server's access key
:param lakefs_secret_key: [required] lakeFS server's secret key
:param lakefs_endpoint: [required] lakeFS server's endpoint address,
       ex: https://example.my-lakefs.com
:param repo: [optional, default = ''] target repository
:param ref: [optional, default = 'main'] target ref (branch name,
       tag, or commit ID)
:param path: [optional, default = ''] target path
N main)r   _LakeFSLoader__lakefs_clientrO   r$   r%   r&   )r   r   r   r   r$   r%   r&   s          r!   r"   LakeFSLoader.__init__D   s[    *  ,/ 
 ,$"*B#d)	 [C2I63s8,B	CI	r1   r(   c                     Xl         g N)r&   )r   r&   s     r!   set_pathLakeFSLoader.set_path`       	r1   c                     Xl         g r[   )r%   )r   r%   s     r!   set_refLakeFSLoader.set_refc   s    r1   c                     Xl         g r[   )r$   )r   r$   s     r!   set_repoLakeFSLoader.set_repof   r^   r1   c                 t   U R                  5         U R                  R                  5       n/ nU R                  R                  U R                  U R
                  U R                  US9nU HK  n[        US   U R                  U R
                  US   U5      nUR                  UR                  5       5        MM     U$ )N)r$   r%   r&   r'      r   )
 _LakeFSLoader__validate_instancerX   rF   r>   r$   r%   r&   UnstructuredLakeFSLoaderextendload)r   	presigneddocsobjsobjlakefs_unstructured_loaders         r!   rj   LakeFSLoader.loadi   s      "((==?	!##..tyy) / 
 C)AA		488SVY*& KK2779:	 
 r1   c                     U R                   b  U R                   S:X  a  [        S5      eU R                  b  U R                  S:X  a  [        S5      eU R                  c  [        S5      eg )NrV   zBno repository was provided. use `set_repo` to specify a repositoryz3no ref was provided. use `set_ref` to specify a refz6no path was provided. use `set_path` to specify a path)r$   r   r%   r&   r   s    r!   __validate_instance LakeFSLoader.__validate_instancew   sc    99		RT  88txx2~RSS99UVV r1   )__lakefs_clientr&   r%   r$   )NrW   rV   )r(   N)rJ   rK   rL   rM   rN   rO   __annotations__r   r"   r\   r`   rc   r   r   rj   rg   rQ   r-   r1   r!   rS   rS   =   s    
I	H
I ## 66 6 	6
 sm6 c]6 sm68S T 3 4 S T d8n Wr1   rS   c                   n   ^  \ rS rSrSr   SS\S\S\S\S\S\4U 4S	 jjjrS
\	4S jr
S
\4S jrSrU =r$ )rh      z(Load from `lakeFS` as unstructured data.urlr$   r%   r&   r'   unstructured_kwargsc                 b   > [         TU ]  " S0 UD6  Xl        X l        X0l        X@l        XPl        g)zInitialize UnstructuredLakeFSLoader.

Args:

:param lakefs_access_key:
:param lakefs_secret_key:
:param lakefs_endpoint:
:param repo:
:param ref:
Nr-   )superr"   ry   r$   r%   r&   r'   )r   ry   r$   r%   r&   r'   rz   	__class__s          r!   r"   !UnstructuredLakeFSLoader.__init__   s/    ( 	/./		r1   r(   c                 J    U R                   U R                  U R                  S.$ )Nr$   r%   r&   r   rr   s    r!   _get_metadata&UnstructuredLakeFSLoader._get_metadata   s    		$((DIIFFr1   c                    SSK Jn  SnU R                  (       a  [        R                  " 5        nU SU R
                  R                  S5      S    3n[        R                  " [        R
                  R                  U5      SS9  [        R                  " U R                  5      nUR                  5         [        USS	9 nUR                  UR                   5        S S S 5        U" US
9sS S S 5        $ U R                  R#                  U5      (       d  [%        S5      eU R                  ['        U5      S  nU" US
9$ ! , (       d  f       Nf= f! , (       d  f       g = f)Nr   )	partitionzlocal://r   T)exist_okwb)mode)filenamez>Non pre-signed URLs are supported only with 'local' blockstore)unstructured.partition.autor   r'   tempfileTemporaryDirectoryr&   splitosmakedirsdirnamer   r   ry   r   openwritecontent
startswithr   len)r   r   local_prefixtemp_dir	file_pathrE   file
local_paths           r!   _get_elements&UnstructuredLakeFSLoader._get_elements   s
   9!<<,,.('j$))//#*>r*B)CD	BGGOOI6F#<<1))+)$/4JJx//0 0 )4 /. $$\22P  #l"3"56Jj11 0/ /.s$   BE?D;E;
E		E
E)r&   r'   r%   r$   ry   )rW   rV   T)rJ   rK   rL   rM   rN   rO   rP   r   r"   dictr   r   r   rQ   __classcell__)r}   s   @r!   rh   rh      st    2   	
    # 6Gt G2t 2 2r1   rh   )r   r   urllib.parser3   typingr   r   r   r   r   langchain_core.documentsr   requests.authr   )langchain_community.document_loaders.baser	   1langchain_community.document_loaders.unstructuredr
   r   rS   rh   r-   r1   r!   <module>r      sO    	   & &    - ' @ T+< +<\BW: BWJ525 52r1   