§
    šŠtjY5  ã                   ó  — d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZ d dlZd dlmZ d dlmZ d dlmZmZ d dlmZ dZd	Zd
ZdZdZdZdZdZdZ  ej!        e"¦  «        Z# eddd¬¦  «         G d„ dee¦  «        ¦   «         Z$dS )é    N)ÚPath)ÚAnyÚDictÚListÚMappingÚOptionalÚSequenceÚUnion)Ú
deprecated)ÚDocument)Ú	BaseModelÚmodel_validator)Ú
BaseLoaderz#{http://www.w3.org/1999/xhtml}tableÚxpathÚidÚsourceÚnameÚ	structureÚtagÚprojectsz#https://api.docugami.com/v1preview1z0.0.24z1.0z!docugami_langchain.DocugamiLoader)ÚsinceÚremovalÚalternative_importc                   ó´  — e Zd ZU dZeZeed<   	 ej	         
                    d¦  «        Zee         ed<   	 dZeed<   	 dZeed<   	 d	Zeed
<   	 dZeed<   	 dZeed<   	 dZeed<   	 dZeed<   	 dZeed<   	 dZee         ed<   	 dZeee                  ed<   	 eeeeef                           ed<   	 dZeed<   	  ed¬¦  «        edeee f         de fd„¦   «         ¦   «         Z!	 	 d*de"dee         d ee#         de$e%         fd!„Z&dede$e         fd"„Z'dede$e         fd#„Z(d$edefd%„Z)	 	 d*d&ededee         d'ee#         de$e%         f
d(„Z*de$e%         fd)„Z+dS )+ÚDocugamiLoaderzdLoad from `Docugami`.

    To use, you should have the ``dgml-utils`` python package installed.
    ÚapiÚDOCUGAMI_API_KEYÚaccess_tokeni   Úmax_text_lengthé    Úmin_text_lengthi   Úmax_metadata_lengthFÚinclude_xml_tagsr   Úparent_hierarchy_levelsÚdoc_idÚparent_id_keyÚsub_chunk_tablesTÚwhitespace_normalize_textNÚ	docset_idÚdocument_idsÚ
file_pathsÚ(include_project_metadata_in_doc_metadataÚbefore)ÚmodeÚvaluesÚreturnc                 ó\  — |                      d¦  «        r$|                      d¦  «        rt          d¦  «        ‚|                      d¦  «        s$|                      d¦  «        st          d¦  «        ‚|                      d¦  «        r$|                      d¦  «        st          d¦  «        ‚|S )zÁValidate that either local file paths are given, or remote API docset ID.

        Args:
            values: The values to validate.

        Returns:
            The validated values.
        r+   r)   z7Cannot specify both file_paths and remote API docset_idz6Must specify either file_paths or remote API docset_idr   z7Must specify access token if using remote API docset_id)ÚgetÚ
ValueError)Úclsr/   s     úk/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/langchain_community/document_loaders/docugami.pyÚvalidate_local_or_remotez'DocugamiLoader.validate_local_or_remoteT   s±   € ð �:Š:�lÑ#Ô#ð 	X¨¯
ª
°;Ñ(?Ô(?ð 	XÝÐVÑWÔWÐWà�zŠz˜,Ñ'Ô'ð 	W°·
²
¸;Ñ0GÔ0Gð 	WÝÐUÑVÔVÐVà�:Š:�kÑ"Ô"ð 	X¨6¯:ª:°nÑ+EÔ+Eð 	XÝÐVÑWÔWÐWàˆó    ÚcontentÚdocument_nameÚadditional_doc_metadatac           	      óø  ‡ ‡‡— 	 ddl m} n# t          $ r t          d¦  «        ‚w xY w	 ddlm} ddlm} n# t          $ r t          d¦  «        ‚w xY wd|dt          fˆˆˆ fd	„}|                     t          j
        |¦  «        ¦  «        }|                     ¦   «         }	 ||	‰ j        ‰ j        ‰ j        ‰ j        ‰ j        ‰ j        ¬
¦  «        }
i }|
D ]†} ||¦  «        }|j                             t(          ¦  «        }|rX|||<   |j        rL ||j        ¦  «        }|j                             t(          ¦  «        }|r|j        r||j        ‰ j        <   |||<   Œ‡t1          |                     ¦   «         ¦  «        S )z?Parse a single DGML document into a list of `Document` objects.r   ©ÚetreeúPCould not import lxml python package. Please install it with `pip install lxml`.)ÚChunk)Ú
get_chunkszaCould not import from dgml-utils python package. Please install it with `pip install dgml-utils`.Údg_chunkr0   c                 óz  •— t          j        | j                             ¦   «         ¦  «                             ¦   «         }t
          | j        t          |t          ‰t          ‰t          | j        t          | j        i}| j        }‰r‰j        r|                     ‰¦  «         t!          |d ‰j        …         |¬¦  «        S )N)Úpage_contentÚmetadata)ÚhashlibÚmd5ÚtextÚencodeÚ	hexdigestÚ	XPATH_KEYr   ÚID_KEYÚDOCUMENT_NAME_KEYÚDOCUMENT_SOURCE_KEYÚSTRUCTURE_KEYr   ÚTAG_KEYr   r,   Úupdater   r   )rA   Ú
_hashed_idrD   rG   r:   r9   Úselfs       €€€r5   Ú_build_framework_chunkz:DocugamiLoader._parse_dgml.<locals>._build_framework_chunk‚   s²   ø€ å œ X¤]×%9Ò%9Ñ%;Ô%;Ñ<Ô<×FÒFÑHÔHˆJå˜8œ>Ý˜
Ý! =Ý# ]Ý˜xÔ1Ý˜œðˆHð ”=ˆDØ&ð =ØÔ@ð =Ø—O’OÐ$;Ñ<Ô<Ð<åØ!Ð"8 DÔ$8Ð"8Ô9Ø!ðñ ô ð r7   )r!   r   r(   r'   r#   r$   )Úlxmlr=   ÚImportErrorÚdgml_utils.modelsr?   Údgml_utils.segmentationr@   r   ÚparseÚioÚBytesIOÚgetrootr!   r   r(   r'   r#   r$   rD   r2   rK   ÚparentrC   r&   Úlistr/   )rR   r8   r9   r:   r=   r?   r@   rS   ÚtreeÚrootÚ	dg_chunksÚframework_chunksrA   Úframework_chunkÚchunk_idÚframework_parent_chunkÚ	parent_ids   ` ``             r5   Ú_parse_dgmlzDocugamiLoader._parse_dgmlj   s   øøø€ ð	Ø"Ð"Ð"Ð"Ð"Ð"Ð"øÝð 	ð 	ð 	Ýð=ñô ð ð	øøøð	Ø/Ð/Ð/Ð/Ð/Ð/Ø:Ð:Ð:Ð:Ð:Ð:Ð:øÝð 	ð 	ð 	ÝðCñô ð ð	øøøð	¨Uð 	µxð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð. �{Š{�2œ: gÑ.Ô.Ñ/Ô/ˆØ�|Š|‰~Œ~ˆà�JØØ Ô0Ø Ô0Ø&*Ô&DØ!Ô2Ø!Ô2Ø$(Ô$@ð
ñ 
ô 
ˆ	ð 13ÐØ!ð 
	Mð 
	MˆHØ4Ð4°XÑ>Ô>ˆOØ&Ô/×3Ò3µFÑ;Ô;ˆHØð MØ-<Ð  Ñ*Ø”?ð MØ-CÐ-CÀHÄOÑ-TÔ-TÐ*Ø 6Ô ?× CÒ CÅFÑ KÔ K�IØ ð MÐ%;Ô%Hð MØGP˜Ô0°Ô1CÑDØ6LÐ(¨Ñ3øåÐ$×+Ò+Ñ-Ô-Ñ.Ô.Ð.s   … Œ&ª7 ·Ac                 ó<  — | j         › d|› d�}g }|rŠt          j        |dd| j        › �i¬¦  «        }|j        rF|                     ¦   «         }|                     |d         ¦  «         |                     dd¦  «        }nt          d	|› d
|j        › d�¦  «        ‚|°Š|S )z1Gets all document details for the given docset IDú	/docsets/z
/documentsÚAuthorizationúBearer )ÚheadersÚ	documentsÚnextNúFailed to download ú
 (status: ú))	r   Úrequestsr2   r   ÚokÚjsonÚextendÚ	ExceptionÚstatus_code)rR   r)   ÚurlÚall_documentsÚresponseÚdatas         r5   Ú_document_details_for_docset_idz.DocugamiLoader._document_details_for_docset_idµ   sØ   € à”Ð9Ð9 IÐ9Ð9Ð9ˆØˆàð 	Ý”|ØØ(Ð*G°DÔ4EÐ*GÐ*GÐHðñ ô ˆHð Œ{ð Ø—}’}‘”�Ø×$Ò$ T¨+Ô%6Ñ7Ô7Ð7Ø—h’h˜v tÑ,Ô,��åØP¨#ÐPÐP¸Ô9MÐPÐPÐPñô ð ð ð 	ð Ðr7   c                 ó>  — | j         › d|› �}g }|rŒt          j        d|dd| j        › �ii ¬¦  «        }|j        rF|                     ¦   «         }|                     |d         ¦  «         |                     dd¦  «        }nt          d	|› d
|j	        › d�¦  «        ‚|°Œ|S )z0Gets all project details for the given docset IDz/projects?docset.id=ÚGETri   rj   ©rk   rz   r   rm   Nrn   ro   rp   )
r   rq   Úrequestr   rr   rs   rt   r2   ru   rv   )rR   r)   rw   Úall_projectsry   rz   s         r5   Ú_project_details_for_docset_idz-DocugamiLoader._project_details_for_docset_idÊ   sÜ   € à”Ð:Ð:¨yÐ:Ð:ˆØˆàð 	ÝÔ'ØØØ(Ð*G°DÔ4EÐ*GÐ*GÐHØð	ñ ô ˆHð Œ{ð Ø—}’}‘”�Ø×#Ò# D¨Ô$4Ñ5Ô5Ð5Ø—h’h˜v tÑ,Ô,��åØP¨#ÐPÐP¸Ô9MÐPÐPÐPñô ð ð ð 	ð  Ðr7   Úprojectc                 ó
  — |                      t          ¦  «        }| j        › d|› d�}g }i }|r™t          j        d|dd| j        › �ii ¬¦  «        }|j        rF|                     ¦   «         }|                     |d         ¦  «         |                      dd	¦  «        }n(|j	        d
k    r|S t          d|› d|j	        › d�¦  «        ‚|°™|D �]¸}|                      d¦  «        }	|                      d¦  «        }
|                      d¦  «        }|	dk    �ro|
�rl|�ri|t                   }i }t          j        d|
› d�dd| j        › �ii ¬¦  «        }|j        �r	 ddlm} n# t          $ r t          d¦  «        ‚w xY w|                     t          j        |j        ¦  «        ¦  «        }|                     ¦   «         }|j        }|                     d|¬¦  «        }|D ]Š}|                     d|¬¦  «        d         j        }d                     |                     d|¬¦  «        d                              ¦   «         ¦  «                             ¦   «         }|d	| j        …         ||<   Œ‹|||<   �Œ¢t          d|
› d�dz   ¦  «        ‚�Œº|S )z#Gets project metadata for all filesz
/projects/z/artifacts/latestr}   ri   rj   r~   Ú	artifactsrm   Ni”  rn   ro   rp   r   rw   Údocumentzreport-values.xmlz/contentr   r<   r>   z
//pr:Entry)Ú
namespacesz./pr:Headingú z
./pr:Valuez	/content z (status: {response.status_code}))r2   rK   r   rq   r   r   rr   rs   rt   rv   ru   rT   r=   rU   rX   rY   rZ   r8   r[   Únsmapr   rG   ÚjoinÚitertextÚstripr"   )rR   r‚   Ú
project_idrw   Úall_artifactsÚper_file_metadatary   rz   ÚartifactÚartifact_nameÚartifact_urlÚartifact_docr%   rD   r=   Úartifact_treeÚartifact_rootÚnsÚentriesÚentryÚheadingÚvalues                         r5   Ú_metadata_for_projectz$DocugamiLoader._metadata_for_projectá   s!  € à—[’[¥Ñ(Ô(ˆ
à”ÐBÐB ZÐBÐBÐBˆØˆà"$ÐØð 	ÝÔ'ØØØ(Ð*G°DÔ4EÐ*GÐ*GÐHØð	ñ ô ˆHð Œ{ð 
Ø—}’}‘”�Ø×$Ò$ T¨+Ô%6Ñ7Ô7Ð7Ø—h’h˜v tÑ,Ô,��ØÔ%¨Ò,Ð,à(Ð(åØP¨#ÐPÐP¸Ô9MÐPÐPÐPñô ð ð ð 	ð& &ð (	ñ (	ˆHØ$ŸLšL¨Ñ0Ô0ˆMØ#Ÿ<š<¨Ñ.Ô.ˆLØ#Ÿ<š<¨
Ñ3Ô3ˆLàÐ 3Ò3Ñ3¸Ñ3ÈÑ3Ø%¥fÔ-�Ø!#�õ $Ô+ØØ#Ð-Ð-Ð-Ø,Ð.K¸Ô8IÐ.KÐ.KÐLØð	ñ ô �ð ”;ñ ðØ.Ð.Ð.Ð.Ð.Ð.Ð.øÝ&ð ð ð Ý)ðIñô ð ðøøøð
 %*§K¢Kµ´
¸8Ô;KÑ0LÔ0LÑ$MÔ$M�MØ$1×$9Ò$9Ñ$;Ô$;�MØ&Ô,�BØ+×1Ò1°,È2Ð1ÑNÔN�GØ!(ð Nð N˜Ø"'§+¢+¨nÈ +Ñ"LÔ"LÈQÔ"OÔ"T˜Ø #§¢Ø!ŸKšK¨À˜KÑDÔDÀQÔG×PÒPÑRÔRñ!ô !çš%™'œ'ð ð -2Ð2L°DÔ4LÐ2LÔ,M˜ Ñ)Ð)Ø08Ð% fÑ-Ñ-å#ØE¨lÐEÐEÐEØ<ñ=ñô ð ùð
 !Ð s   ÅEÅE6Údocument_idÚadditional_metadatac                 óä   — | j         › d|› d|› d�}t          j        d|dd| j        › �ii ¬¦  «        }|j        r|                      |j        ||¬¦  «        S t          d	|› d
|j        › d�¦  «        ‚)zLoad chunks for a document.rh   z/documents/z/dgmlr}   ri   rj   r~   )r8   r9   r:   rn   ro   rp   )	r   rq   r   r   rr   rf   r8   ru   rv   )rR   r›   r)   r9   rœ   rw   ry   s          r5   Ú_load_chunks_for_documentz(DocugamiLoader._load_chunks_for_document(  sº   € ð ”ÐLÐL IÐLÐL¸+ÐLÐLÐLˆåÔ#ØØØ$Ð&C°Ô0AÐ&CÐ&CÐDØð	
ñ 
ô 
ˆð Œ;ð 		Ø×#Ò#Ø Ô(Ø+Ø(;ð $ñ ô ð õ ØL cÐLÐL°XÔ5IÐLÐLÐLñô ð r7   c                 ó,  ‡ — g }‰ j         �r‰ j        �r‰                      ‰ j        ¦  «        }‰ j        rˆ fd„|D ¦   «         }‰                      ‰ j        ¦  «        }i }|rW‰ j        rP|D ]M}‰                      |¦  «        }|D ]3}||vr||         ||<   Œ||                              ||         ¦  «         Œ4ŒN|D ]_}|t                   }	| 	                    t          ¦  «        }
| 	                    |	¦  «        }|‰                      |	‰ j        |
|¬¦  «        z  }Œ`ny‰ j        rr‰ j        D ]j}t          |¦  «        }t          |d¦  «        5 }|‰                      |                     ¦   «         |j        ¬¦  «        z  }ddd¦  «         n# 1 swxY w Y   Œk|S )zLoad documents.c                 ó<   •— g | ]}|t                    ‰j        v ¯|‘ŒS © )rK   r*   )Ú.0ÚdrR   s     €r5   ú
<listcomp>z'DocugamiLoader.load.<locals>.<listcomp>L  s1   ø€ ð %ð %ð %Ø°Aµf´IÀÔARÐ4RÐ4R�AÐ4RÐ4RÐ4Rr7   )r›   r)   r9   rœ   Úrb)r8   r9   N)r   r)   r{   r*   r�   r,   rš   rP   rK   r2   rL   rž   r+   r   Úopenrf   Úreadr   )rR   ÚchunksÚ_document_detailsÚ_project_detailsÚcombined_project_metadatar‚   rD   Úfile_idÚdocr%   Údoc_nameÚdoc_metadataÚpathÚfiles   `             r5   ÚloadzDocugamiLoader.loadD  sM  ø€ à!#ˆàÔñ '	 ¤ñ '	à $× DÒ DÀTÄ^Ñ TÔ TÐØÔ ð ð%ð %ð %ð %Ø0ð%ñ %ô %Ð!ð  $×BÒBÀ4Ä>ÑRÔRÐØ9;Ð%Øð 	Y DÔ$Qð 	Yð  0ð Yð Y�GØ#×9Ò9¸'ÑBÔB�HØ#+ð Yð Y˜Ø"Ð*CÐCÐCØAIÈ'ÔARÐ5°gÑ>Ð>à5°gÔ>×EÒEÀhÈwÔFWÑXÔXÐXÐXð	Yð )ð 	ð 	�Ø�Vœ�ØŸ7š7Õ#4Ñ5Ô5�Ø8×<Ò<¸VÑDÔD�Ø˜$×8Ò8Ø &Ø"œnØ"*Ø(4ð	 9ñ ô ñ ��ð		ð Œ_ð 	àœð ð �Ý˜D‘z”z�Ý˜$ Ñ%Ô%ð ¨Ø˜d×.Ò.Ø $§	¢	¡¤Ø&*¤ið /ñ ô ñ �Fðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð øð ˆs   Å
2FÆF	ÆF	)NN),Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚDEFAULT_API_ENDPOINTr   ÚstrÚ__annotations__ÚosÚenvironr2   r   r   r   Úintr!   r"   r#   Úboolr$   r&   r'   r(   r)   r*   r	   r
   r   r,   r   Úclassmethodr   r   r6   Úbytesr   r   r   rf   r{   r�   rš   rž   r²   r¡   r7   r5   r   r      s  € € € € € € ðð ð
 $€CˆÐ#Ð#Ñ#Ø+à"$¤*§.¢.Ð1CÑ"DÔ"D€L�(˜3”-ÐDÐDÑDØ/à€O�SÐÐÑØ,à€O�SÐÐÑØSà"Ð˜Ð"Ð"Ñ"Ø/à"Ð�dÐ"Ð"Ñ"Ø8à#$Ð˜SÐ$Ð$Ñ$ØKà!€M�3Ð!Ð!Ñ!Ø)à"Ð�dÐ"Ð"Ñ"Ø9à&*Ð˜tÐ*Ð*Ñ*ð'ð  $€Iˆx˜Œ}Ð#Ð#Ñ#Ø,à,0€L�(˜8 Cœ=Ô)Ð0Ð0Ñ0Ø/à˜ %¨¨c¨	Ô"2Ô3Ô4Ð4Ð4Ñ4Ø&à59Ð,¨dÐ9Ð9Ñ9ØVà€_˜(Ð#Ñ#Ô#Øð¨d°3¸°8¬nð Àð ð ð ñ „[ñ $Ô#ðð. (,Ø59ð	I/ð I/àðI/ð   ”}ðI/ð "*¨'Ô!2ð	I/ð
 
ˆhŒðI/ð I/ð I/ð I/ðV¸ð ÀÀdÄð ð ð ð ð*¸ð ÀÀTÄ
ð ð ð ð ð.E!¨Tð E!°dð E!ð E!ð E!ð E!ðV (,Ø15ðð àðð ðð   ”}ð	ð
 & gÔ.ðð 
ˆhŒðð ð ð ð8-�d˜8”nð -ð -ð -ð -ð -ð -r7   r   )%rE   rY   Úloggingrº   Úpathlibr   Útypingr   r   r   r   r   r	   r
   rq   Úlangchain_core._api.deprecationr   Úlangchain_core.documentsr   Úpydanticr   r   Ú)langchain_community.document_loaders.baser   Ú
TABLE_NAMErJ   rK   rM   rL   rN   rO   ÚPROJECTS_KEYr·   Ú	getLoggerr³   Úloggerr   r¡   r7   r5   ú<module>rË      ss  ðØ €€€Ø 	€	€	€	Ø €€€Ø 	€	€	€	Ø Ð Ð Ð Ð Ð Ø FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ FÐ Fà €€€Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø -Ð -Ð -Ð -Ð -Ð -Ø /Ð /Ð /Ð /Ð /Ð /Ð /Ð /à @Ð @Ð @Ð @Ð @Ð @à2€
à€	Ø	€ØÐ ØÐ Ø€Ø
€Ø€à<Ð à	ˆÔ	˜8Ñ	$Ô	$€ð €Ø
ØØ:ðñ ô ð
Nð Nð Nð Nð N�Z ñ Nô Nñô ð
Nð Nð Nr7   