o
    sh_K                     @  s   d dl mZ d dlmZ d dlmZ d dlmZmZm	Z	m
Z
mZmZmZ d dlZd dlZd dlmZmZ d dlmZmZ d dlmZ d d	lmZ G d
d dZdddZG dd dejZdS )    )annotations)nullcontext)partial)AnyDictIterableIteratorListOptionalTupleN)Tensornn)get_device_statesset_device_states)SentenceTransformer)Transformerc                   @  s.   e Zd ZdZdddZdddZddd	Zd
S )RandContexta  
    Random-state context manager class. Reference: https://github.com/luyug/GradCache.

    This class will back up the pytorch's random state during initialization. Then when the context is activated,
    the class will set up the random state with the backed-up one.
    returnNonec                 G  s   t  | _t| \| _| _d S N)torchget_rng_statefwd_cpu_stater   fwd_gpu_devicesfwd_gpu_states)selftensors r   n/var/www/html/alpaca_bot/venv/lib/python3.10/site-packages/sentence_transformers/losses/CachedGISTEmbedLoss.py__init__   s   
zRandContext.__init__c                 C  s<   t jj| jdd| _| j  t | j t| j| j	 d S )NT)devicesenabled)
r   randomfork_rngr   _fork	__enter__set_rng_stater   r   r   r   r   r   r   r%      s   
zRandContext.__enter__c                 C  s   | j ||| d | _ d S r   )r$   __exit__)r   exc_typeexc_valexc_tbr   r   r   r(   "   s   
zRandContext.__exit__N)r   r   )__name__
__module____qualname____doc__r   r%   r(   r   r   r   r   r      s
    

r   grad_outputr   sentence_featuresIterable[Dict[str, Tensor]]loss_objCachedGISTEmbedLossr   r   c           
   	   C  s   |j dusJ |jdusJ t ; t||j |jD ])\}}}t|j|dd|d|D ]\\}}}}t| | |  }	|	  q,qW d   dS 1 sPw   Y  dS )zOA backward hook to backpropagate the cached gradients mini-batch by mini-batch.NTF)sentence_feature	with_gradcopy_random_staterandom_states)	cacher8   r   enable_gradzipembed_minibatch_iterdotflattenbackward)
r0   r1   r3   r5   gradr8   reps_mb_grad_mb	surrogater   r   r   _backward_hook'   s$   
	
"rE   c                      st   e Zd Z			d6d7 fddZd8ddZ	d9d:d d!Z	d9d;d%d&Zd<d*d+Zd<d,d-Zd=d1d2Z	d>d4d5Z
  ZS )?r4   {Gz?    Fmodelr   guidetemperaturefloatmini_batch_sizeintshow_progress_barboolr   r   c                   s   t t|   || _|| _|| _tjdd| _t	|d t
r%t	|d t
s)tdt | _|| _d| _d| _|| _|jj|jjkpG|j|jk | _dS )a  
        This loss is a combination of :class:`GISTEmbedLoss` and :class:`CachedMultipleNegativesRankingLoss`.
        Typically, :class:`MultipleNegativesRankingLoss` requires a larger batch size for better performance.
        :class:`GISTEmbedLoss` yields stronger training signals than :class:`MultipleNegativesRankingLoss` due to the
        use of a guide model for in-batch negative sample selection. Meanwhile, :class:`CachedMultipleNegativesRankingLoss`
        allows for scaling of the batch size by dividing the computation into two stages of embedding and loss
        calculation, which both can be scaled by mini-batches (https://arxiv.org/pdf/2101.06983.pdf).

        By combining the guided selection from :class:`GISTEmbedLoss` and Gradient Cache from
        :class:`CachedMultipleNegativesRankingLoss`, it is possible to reduce memory usage while maintaining performance
        levels comparable to those of :class:`GISTEmbedLoss`.

        Args:
            model: SentenceTransformer model
            guide: SentenceTransformer model to guide the in-batch negative sample selection.
            temperature: Temperature parameter to scale the cosine similarities.
            mini_batch_size: Mini-batch size for the forward pass, this denotes how much memory is actually used during
                training and evaluation. The larger the mini-batch size, the more memory efficient the training is, but
                the slower the training will be. It's recommended to set it as high as your GPU memory allows. The default
                value is 32.
            show_progress_bar: If True, a progress bar for the mini-batches is shown during training. The default is False.

        References:
            - Efficient Natural Language Response Suggestion for Smart Reply, Section 4.4: https://arxiv.org/pdf/1705.00652.pdf
            - Scaling Deep Contrastive Learning Batch Size under Memory Limited Setup: https://arxiv.org/pdf/2101.06983.pdf
            - GISTEmbed: Guided In-sample Selection of Training Negatives for Text Embedding Fine-tuning https://arxiv.org/abs/2402.16829

        Requirements:
            1. (anchor, positive) pairs or (anchor, positive, negative pairs)
            2. Should be used with large batch sizes for superior performance, but has slower training time than :class:`MultipleNegativesRankingLoss`

        Relations:
            - Equivalent to :class:`GISTEmbedLoss`, but with caching that allows for much higher batch sizes

        Inputs:
            +---------------------------------------+--------+
            | Texts                                 | Labels |
            +=======================================+========+
            | (anchor, positive) pairs              | none   |
            +---------------------------------------+--------+
            | (anchor, positive, negative) triplets | none   |
            +---------------------------------------+--------+

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                guide = SentenceTransformer("all-MiniLM-L6-v2")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.CachedGISTEmbedLoss(model, guide, mini_batch_size=64)

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()
        dimr   z_Both the training model and the guiding model must be based on the `transformers` architecture.N)superr4   r   rH   rI   rJ   r   CosineSimilaritysimilarity_fct
isinstancer   
ValueErrorCrossEntropyLosscross_entropy_lossrL   r9   r8   rN   	tokenizervocabmax_seq_lengthmust_retokenize)r   rH   rI   rJ   rL   rN   	__class__r   r   r   ?   s    H
zCachedGISTEmbedLoss.__init__embed1r   embed2c                 C  s   |  |d|dS )N   r   )rU   	unsqueeze)r   r`   ra   r   r   r   
sim_matrix   s   zCachedGISTEmbedLoss.sim_matrixNr5   Dict[str, Tensor]beginendr6   r7   random_stateOptional[RandContext]$Tuple[Tensor, Optional[RandContext]]c              	     s"  |rt ntj}|du rt  n|} fdd| D }	|f |  |r+t|	  nd}|	d }
W d   n1 s>w   Y  t . jrgjjj	|	d dd}j
|}	fdd|	 D }	
|	d }W d   n1 sxw   Y  W d   n1 sw   Y  |
||fS )	zYDo forward pass on a minibatch of the input features and return corresponding embeddings.Nc                   s   i | ]\}}||  qS r   r   ).0kv)rf   rg   r   r   
<dictcomp>   s    z7CachedGISTEmbedLoss.embed_minibatch.<locals>.<dictcomp>sentence_embedding	input_idsT)skip_special_tokensc                   s    i | ]\}}||  jjqS r   )torI   device)rk   keyvaluer'   r   r   rn      s    )r   r   no_graditemsr   valuesrH   r]   rZ   batch_decoderI   tokenize)r   r5   rf   rg   r6   r7   rh   grad_contextrandom_state_contextsentence_feature_minibatchrepsdecoded
guide_repsr   )rf   rg   r   r   embed_minibatch   s,   



z#CachedGISTEmbedLoss.embed_minibatchr8   Optional[List[RandContext]].Iterator[Tuple[Tensor, Optional[RandContext]]]c              	   c  s    |d }|j \}}ttjd|| jd| j dD ]%\}}	|	| j }
| j||	|
|||du r/dn|| d\}}}|||fV  qdS )z`Do forward pass on all the minibatches of the input features and yield corresponding embeddings.rp   r   zEmbed mini-batchesdescdisableN)r5   rf   rg   r6   r7   rh   )shape	enumeratetqdmtrangerL   rN   r   )r   r5   r6   r7   r8   rp   bszrB   iber~   r   rh   r   r   r   r<      s.   

	z(CachedGISTEmbedLoss.embed_minibatch_iterr~   List[List[Tensor]]reps_guidedc                 C  s  t |dkr|\}}|\}}d}d}nt |dkr$|\}}}|\}}}n	tdt |tj|dd}tj|dd}tj|dd}tj|dd}|rYtj|dd}tj|dd}t|d |j	}	|j
d }
g }tjd|
| jd| j dD ]}|| j }| ||| |}| ||| |}| ||| |}|j|d	d
d}| ||| |}| ||| |}| ||| |}tj |||k< tj |||k< tj |||k< tj|||gdd}|dur| ||| |}| ||| |}tj |||k< tj||gdd}|| j }| ||	|| t | |
 }|  ||  q{t| }dd |D | _|S )zMCalculate the cross-entropy loss and cache the gradients wrt. the embeddings.   N   "Expected 2 or 3 embeddings, got {}r   rQ   Preparing cachesr   offsetrP   rb   c                 S  s   g | ]	}d d |D qS )c                 S  s   g | ]}|j qS r   )r@   )rk   rr   r   r   
<listcomp>  s    zUCachedGISTEmbedLoss.calculate_loss_and_cache_gradients.<locals>.<listcomp>.<listcomp>r   )rk   rsr   r   r   r     s    zJCachedGISTEmbedLoss.calculate_loss_and_cache_gradients.<locals>.<listcomp>)lenrW   formatr   catarangesizelongrr   rs   r   r   r   rL   rN   rd   diagonalviewinfrJ   rY   r?   appenddetachsumrequires_grad_r9   r   r~   r   anchorpositiveanchor_guidepositive_guidenegativenegative_guidelabels
batch_sizelossesr   r   guided_ap_simguided_aa_simguided_pp_sim
guided_simap_simaa_simpp_simscoresguided_an_siman_simloss_mbatchlossr   r   r   "calculate_loss_and_cache_gradients   sd   





 z6CachedGISTEmbedLoss.calculate_loss_and_cache_gradientsc                 C  sj  t |dkr|\}}|\}}d}d}nt |dkr$|\}}}|\}}}n	tdt |tj|dd}tj|dd}tj|dd}tj|dd}|rYtj|dd}tj|dd}t|d |j	}	|j
d }
g }tjd|
| jd| j dD ]}|| j }| ||| |}| ||| |}| ||| |}|j|d	d
d}| ||| |}| ||| |}| ||| |}tj |||k< tj |||k< tj |||k< tj|||gdd}|dur| ||| |}| ||| |}tj |||k< tj||gdd}|| j }| ||	|| t | |
 }|| q{t|}|S )zACalculate the cross-entropy loss. No need to cache the gradients.r   Nr   r   r   rQ   r   r   r   rP   rb   )r   rW   r   r   r   r   r   r   rr   rs   r   r   r   rL   rN   rd   r   r   r   rJ   rY   r   r   r   r   r   r   calculate_loss"  s`   





 z"CachedGISTEmbedLoss.calculate_lossr1   r2   r   c                 C  s   g }g }g | _ |D ];}g }g }g }| j|dddD ]\}	}
}||	   ||
  || q|| || | j | q	t r[| ||}|t	t
|| d |S | ||}|S )NFT)r5   r6   r7   )r1   r3   )r8   r<   r   r   r   r   is_grad_enabledr   register_hookr   rE   r   )r   r1   r   r~   r   r5   reps_mbsreps_guided_mbsrandom_state_mbsrA   reps_guided_mbrh   r   r   r   r   forwardh  s0   

zCachedGISTEmbedLoss.forwardDict[str, Any]c                 C  s   | j | jdS )NrI   rJ   r   r'   r   r   r   get_config_dict  s   z#CachedGISTEmbedLoss.get_config_dict)rF   rG   F)rH   r   rI   r   rJ   rK   rL   rM   rN   rO   r   r   )r`   r   ra   r   r   r   r   )r5   re   rf   rM   rg   rM   r6   rO   r7   rO   rh   ri   r   rj   )
r5   re   r6   rO   r7   rO   r8   r   r   r   )r~   r   r   r   r   r   )r1   r2   r   r   r   r   )r   r   )r,   r-   r.   r   rd   r   r<   r   r   r   r   __classcell__r   r   r^   r   r4   >   s    
Z
#

J
F )r0   r   r1   r2   r3   r4   r   r   )
__future__r   
contextlibr   	functoolsr   typingr   r   r   r   r	   r
   r   r   r   r   r   torch.utils.checkpointr   r   sentence_transformersr   sentence_transformers.modelsr   r   rE   Moduler4   r   r   r   r   <module>   s    $
