§
    éƒtj´  ã                  óØ   — d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ dd	lmZmZmZ e G d
„ d¦  «        ¦   «         Zd.d„Zd/d0d„Zd1d2d„Zd3d4d „Zd5d$„Z	 d6dd%œd7d-„ZdS )8zKHybrid retrieval: BM25 + dense vectors with RRF fusion (+ optional rerank).é    )ÚannotationsN)Ú	dataclass)ÚAny)ÚDocument)Úget_settings)Ú	rag_debug)Úget_agent_vectorstoreÚnormalize_whitespaceÚretrieve_agent_scoredc                  ó@   — e Zd ZU ded<   ded<   dZded<   dZded<   dS )	Ú	HybridHitr   ÚdocÚfloatÚscoreNzfloat | NoneÚdense_scoreÚ
bm25_score)Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r   r   © ó    ú*/var/www/html/CA-Chatbot/app/fdi/hybrid.pyr   r      sF   € € € € € € à€M€M�MØ€L€L�LØ $€KÐ$Ð$Ð$Ñ$Ø#€JÐ#Ð#Ð#Ñ#Ð#Ð#r   r   ÚtextÚstrÚreturnú	list[str]c                óT   — t          j        d| pd                     ¦   «         ¦  «        S )Nz[a-z0-9]{2,}Ú )ÚreÚfindallÚlower)r   s    r   Ú	_tokenizer#      s$   € ÝŒ:�o¨¨
°×'9Ò'9Ñ';Ô';Ñ<Ô<Ð<r   é<   ÚrankÚintÚkr   c                ó   — d|| z   z  S )Ng      ð?r   )r%   r'   s     r   Ú_rrfr)      s   € Ø�!�d‘(ÑÐr   éÐ  Ústorer   Úlimitúlist[tuple[str, str, dict]]c           	     ó`  — 	 t          | j                             ¦   «         pd¦  «        }n# t          $ r g cY S w xY w|dk    rg S t	          ||¦  «        }| j                             ddg|¬¦  «        }|                     d¦  «        pg }|                     d¦  «        pg }|                     d¦  «        pi gt          |¦  «        z  }g }t          |¦  «        D ]S\  }	}
|                     |
|	t          |¦  «        k     r||	         nd|	t          |¦  «        k     r||	         ni f¦  «         ŒT|S )z:Load a working set of chunks for BM25 (capped for memory).r   Ú	documentsÚ	metadatas)Úincluder,   Úidsr   )	r&   Ú_collectionÚcountÚ	ExceptionÚminÚgetÚlenÚ	enumerateÚappend)r+   r,   r4   ÚnÚgotr2   ÚdocsÚmetasÚoutÚiÚdoc_ids              r   Ú_collection_docsrB       sF  € ðÝ�EÔ%×+Ò+Ñ-Ô-Ð2°Ñ3Ô3ˆˆøÝð ð ð Øˆ	ˆ	ˆ	ðøøøà�‚z€zØˆ	ÝˆE�5ÑÔ€AØ
Ô
×
Ò
¨°kÐ(BÈ!Ð
Ñ
LÔ
L€CØ
�'Š'�%‰.Œ.Ð
˜B€CØ�7Š7�;ÑÔÐ% 2€DØ�GŠG�KÑ Ô Ð3 R D­3¨s©8¬8¡O€EØ
€CÝ˜s‘^”^ð cð c‰	ˆˆ6Ø�
Š
�F q­3¨t©9¬9¢} }˜D œG˜G¸"È!ÍcÐRWÉjÌjÊ.È.¸eÀA¼h¸hÐ^`ÐaÑbÔbÐbÐbØ€Js   ‚(+ «:¹:é   Úqueryúlist[tuple[Document, float]]c                ó†  ‡— 	 ddl m} n&# t          $ r t          j        d¦  «         g cY S w xY wt          | ¦  «        }|sg S d„ |D ¦   «         }t          |¦  «        sg S  ||¦  «        }|                     t          |¦  «        ¦  «        Št          t          t          ‰¦  «        ¦  «        ˆfd„¬¦  «        d |…         }g }|D ]y}	t          ‰|	         ¦  «        }
|
dk    rŒ||	         \  }}}t          |pi ¦  «        }|                     d|¦  «         |                     t!          |pd|¬	¦  «        |
f¦  «         Œz|S )
Nr   )Ú	BM25Okapiu)   rank_bm25 not installed â€” BM25 disabledc                ó4   — g | ]\  }}}t          |¦  «        ‘ŒS r   )r#   )Ú.0Ú_r   s      r   ú
<listcomp>zbm25_search.<locals>.<listcomp>=   s$   € Ð<Ð<Ð<©¨¨D°!•Y˜t‘_”_Ð<Ð<Ð<r   c                ó0   •— t          ‰|          ¦  «         S ©N)r   )r@   Úscoress    €r   ú<lambda>zbm25_search.<locals>.<lambda>B   s   ø€ µu¸VÀA¼YÑ7GÔ7GÐ6G€ r   ©ÚkeyÚ	record_idr   )Úpage_contentÚmetadata)Ú	rank_bm25rG   ÚImportErrorÚdbgÚlog_warnrB   ÚanyÚ
get_scoresr#   ÚsortedÚranger8   r   ÚdictÚ
setdefaultr:   r   )r+   rD   r'   rG   ÚrowsÚcorpus_tokensÚbm25Úrankedr?   r@   ÚscÚ_idr   ÚmetarN   s                 @r   Úbm25_searchrf   3   sˆ  ø€ ðØ'Ð'Ð'Ð'Ð'Ð'Ð'øÝð ð ð ÝŒÐ@ÑAÔAÐAØˆ	ˆ	ˆ	ðøøøõ ˜EÑ"Ô"€DØð Øˆ	Ø<Ð<°tÐ<Ñ<Ô<€MÝˆ}ÑÔð Øˆ	Øˆ9�]Ñ#Ô#€DØ�_Š_�Y uÑ-Ô-Ñ.Ô.€FÝ•E�#˜f™+œ+Ñ&Ô&Ð,GÐ,GÐ,GÐ,GÐHÑHÔHÈÈ!ÈÔL€FØ(*€CØð Kð KˆÝ�6˜!”9ÑÔˆØ�Š7ˆ7ØØ˜qœ'‰ˆˆT�4Ý�D�J˜BÑÔˆØ�Š˜ SÑ)Ô)Ð)Ø�
Š
•H¨$¨*°"¸tÐDÑDÔDÀbÐIÑJÔJÐJÐJØ€Js   ƒ
 Š -¬-Úhitsúlist[HybridHit]Útop_nc                ó0  ‡ — t          ¦   «         j        rt          |¦  «        dk    r
|d|…         S 	 ddlm} n# t
          $ r |d|…         cY S w xY w	 t          t          dd¦  «        }|€ |d¦  «        }|t          _        ˆ fd„|D ¦   «         }| 	                    |¦  «        }d„ t          ||¦  «        D ¦   «         }|                     d	„ ¬
¦  «         |d|…         S # t
          $ r)}t          j        d|¦  «         |d|…         cY d}~S d}~ww xY w)z6Cross-encoder rerank when enabled; otherwise identity.é   Nr   )ÚCrossEncoderÚ_modelz$cross-encoder/ms-marco-MiniLM-L-6-v2c                ó<   •— g | ]}‰|j         j        d d…         f‘ŒS )Nr*   )r   rS   )rI   ÚhrD   s     €r   rK   z#optional_rerank.<locals>.<listcomp>]   s,   ø€ ÐBÐBÐB¸�%˜œÔ+¨E¨T¨EÔ2Ð3ÐBÐBÐBr   c                óp   — g | ]3\  }}t          |j        t          |¦  «        |j        |j        ¦  «        ‘Œ4S r   )r   r   r   r   r   )rI   ro   Úss      r   rK   z#optional_rerank.<locals>.<listcomp>_   sF   € ð 
ð 
ð 
á��1õ �a”e�U 1™XœX q¤}°a´lÑCÔCð
ð 
ð 
r   c                ó   — | j          S rM   ©r   )Úxs    r   rO   z!optional_rerank.<locals>.<lambda>c   s
   €  Q¤W H€ r   rP   zrerank failed: %s)r   Ú
fdi_rerankr8   Úsentence_transformersrl   r5   ÚgetattrÚoptional_rerankrm   ÚpredictÚzipÚsortrW   rX   )	rD   rg   ri   rl   ÚmodelÚpairsrN   ÚrescoredÚexcs	   `        r   rx   rx   O   st  ø€ å‰>Œ>Ô$ð ­¨D©	¬	°Qª¨Ø�F�U�FŒ|ÐðØ6Ð6Ð6Ð6Ð6Ð6Ð6øÝð ð ð Ø�F�U�FŒ|ÐÐÐðøøøðå�¨°4Ñ8Ô8ˆØˆ=Ø �LÐ!GÑHÔHˆEØ%*�OÔ"ØBÐBÐBÐB¸TÐBÑBÔBˆØ—’˜uÑ%Ô%ˆð
ð 
å˜D &Ñ)Ô)ð
ñ 
ô 
ˆð 	�ŠÐ,Ð,ˆÑ-Ô-Ð-Ø˜˜˜ÔÐøÝð ð ð ÝŒÐ(¨#Ñ.Ô.Ð.Ø�F�U�FŒ|ÐÐÐÐÐÐøøøøðøøøs.   ³: ºAÁAÁBC" Ã"
DÃ,DÄ
DÄD)Úchunk_kindsÚorg_idÚagent_idÚquestionú
int | Noner€   ú"tuple[str, ...] | list[str] | Noneú6tuple[list[tuple[Document, float | None]], str | None]c          	     óf  — t          ¦   «         }t          |¦  «        }|sg dfS |p|j        }t          |dz  d¦  «        }t	          | ||||¬¦  «        \  }}	t          | |¦  «        }
|j        rt          |
||¬¦  «        ng }|s|sg |	pdfS i }dd„}t          |d¬¦  «        D ]U\  }\  }} ||¦  «        }| 	                    ||ddddœ¦  «        }|dxx         t          |¦  «        z  cc<   ||d<   ||d<   ŒVt          |d¬¦  «        D ]j\  }\  }} ||¦  «        }| 	                    ||ddddœ¦  «        }|dxx         t          |¦  «        z  cc<   ||d<   |                     d¦  «        €||d<   Œkd„ |                     ¦   «         D ¦   «         }|                     d„ ¬¦  «         t          ||dt          |dz  d¦  «        …         |¦  «        }d„ |d|…         D ¦   «         }t          j        d|dd…         t#          |¦  «        t#          |¦  «        t#          |¦  «        ¦  «         |dfS )z>Fuse dense Chroma scores with BM25 via reciprocal rank fusion.Úempty_queryé   é   )r'   r€   )r'   Úno_hybrid_hitsr   r   r   r   c                óº   — | j         pi }t          |                     d¦  «        p0|                     d¦  «        pt          | j        d d…         ¦  «        ¦  «        S )NrR   Úchunk_idéÈ   )rT   r   r7   ÚhashrS   )r   re   s     r   Úkey_forz hybrid_retrieve.<locals>.key_for‡   sU   € ØŒ|Ð!˜rˆÝ�4—8’8˜KÑ(Ô(Ð`¨D¯HªH°ZÑ,@Ô,@Ð`ÅDÈÔIYÐZ^Ð[^ÐZ^ÔI_ÑD`ÔD`ÑaÔaÐar   rk   )Ústartg        N©r   ÚrrfÚdensera   r“   r”   ra   c           
     ó°   — g | ]S}t          |d          t          |d         ¦  «        |                     d¦  «        |                     d¦  «        ¦  «        ‘ŒTS r’   )r   r   r7   )rI   Úslots     r   rK   z#hybrid_retrieve.<locals>.<listcomp>š   s^   € ð ð ð àõ 	�$�u”+�u T¨%¤[Ñ1Ô1°4·8²8¸GÑ3DÔ3DÀdÇhÂhÈvÑFVÔFVÑWÔWðð ð r   c                ó   — | j          S rM   rs   )ro   s    r   rO   z!hybrid_retrieve.<locals>.<lambda>ž   s
   € ˜aœg˜X€ r   rP   é   é   c                ó*   — g | ]}|j         |j        f‘ŒS r   )r   r   )rI   ro   s     r   rK   z#hybrid_retrieve.<locals>.<listcomp>¡   s!   € Ð
3Ð
3Ð
3 ˆAŒE�1”7ÐÐ
3Ð
3Ð
3r   z.hybrid_retrieve q=%r dense=%s bm25=%s fused=%séP   )r   r   r   r   )r   r
   Úkb_retrieval_top_kÚmaxr   r	   Úfdi_hybrid_searchrf   r9   r^   r)   r7   Úvaluesr{   rx   rW   Úlog_infor8   )r�   r‚   rƒ   r'   r€   ÚsettingsÚtop_kÚdense_kr”   Úreasonr+   Úsparser“   r�   r%   r   r   rQ   r–   Úfusedr?   s                        r   Úhybrid_retriever§   j   sÇ  € õ ‰~Œ~€HÝ# HÑ-Ô-€HØð !Ø�=Ð Ð àÐ,�Ô,€EÝ�%˜!‘)˜RÑ Ô €GÝ)Ø�˜( g¸;ðñ ô �M€Eˆ6õ " &¨(Ñ3Ô3€EØ8@Ô8RÐZ�[˜ ¨GÐ4Ñ4Ô4Ð4ÐXZ€Fàð .˜ð .Ø�6Ð-Ð-Ð-Ð-ð &(€Cðbð bð bð bõ (¨°QÐ7Ñ7Ô7ð ð Ñˆ‰lˆs�EØˆg�c‰lŒlˆØ�~Š~˜c¨3°sÀTÐSWÐ#XÐ#XÑYÔYˆØˆUˆˆŒ•t˜D‘z”zÑ!ˆˆ‰ØˆˆW‰ØˆˆU‰ˆå'¨°aÐ8Ñ8Ô8ð ð Ñˆ‰lˆs�EØˆg�c‰lŒlˆØ�~Š~˜c¨3°sÀTÐSWÐ#XÐ#XÑYÔYˆØˆUˆˆŒ•t˜D‘z”zÑ!ˆˆ‰ØˆˆV‰Ø�8Š8�GÑÔÐ$ØˆD�‰Køðð à—J’J‘L”Lðñ ô €Eð 
‡J‚JÐ%Ð%€JÑ&Ô&Ð&Ý˜H eÐ,@­c°%¸!±)¸RÑ.@Ô.@Ð,@Ô&AÀ5ÑIÔI€Eà
3Ð
3 U¨6¨E¨6¤]Ð
3Ñ
3Ô
3€CÝ„LØ8Ø��"�ŒÝˆE‰
Œ
ÝˆF‰ŒÝˆC‰Œñô ð ð �ˆ9Ðr   )r   r   r   r   )r$   )r%   r&   r'   r&   r   r   )r*   )r+   r   r,   r&   r   r-   )rC   )r+   r   rD   r   r'   r&   r   rE   )rD   r   rg   rh   ri   r&   r   rh   rM   )r�   r   r‚   r   rƒ   r   r'   r„   r€   r…   r   r†   )Ú__doc__Ú
__future__r   r    Údataclassesr   Útypingr   Úlangchain_core.documentsr   Ú
app.configr   Úapp.saasr   rW   Úapp.saas.kbr	   r
   r   r   r#   r)   rB   rf   rx   r§   r   r   r   ú<module>r°      s|  ðØ QÐ Qà "Ð "Ð "Ð "Ð "Ð "à 	€	€	€	Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð à -Ð -Ð -Ð -Ð -Ð -à #Ð #Ð #Ð #Ð #Ð #Ø %Ð %Ð %Ð %Ð %Ð %Ø ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ ZÐ Zð ð$ð $ð $ð $ð $ñ $ô $ñ „ð$ð=ð =ð =ð =ðð ð ð ð ðð ð ð ð ð&ð ð ð ð ð8ð ð ð ð> ð	?ð 7;ð?ð ?ð ?ð ?ð ?ð ?ð ?ð ?r   