o
    sh*                     @   sZ   d dl mZmZmZ d dlZd dlmZmZ d dlmZ d dl	m
Z
 G dd dejZdS )    )AnyDictIterableN)Tensornn)Transformer)SentenceTransformerc                	       s   e Zd Z	ddedededdf fddZd	ed
edefddZdee	e
ef  dedefddZde	e
ef fddZede
fddZ  ZS )GISTEmbedLoss{Gz?modelguidetemperaturereturnNc                    st   t t|   || _|| _|| _tjdd| _t	|d t
r%t	|d t
s)td|jj|jjkp6|j|jk | _dS )a	  
        This loss is used to train a SentenceTransformer model using the GISTEmbed algorithm.
        It takes a model and a guide model as input, and uses the guide model to guide the
        in-batch negative sample selection. The cosine similarity is used to compute the loss
        and the temperature parameter is used to scale the cosine similarities.

        Args:
            model: SentenceTransformer model based on a `transformers`
                model.
            guide: SentenceTransformer model to guide the in-batch
                negative sample selection.
            temperature: Temperature parameter to scale the cosine
                similarities.

        References:
            - For further details, see: https://arxiv.org/abs/2402.16829

        Requirements:
            1. (anchor, positive, negative) triplets
            2. (anchor, positive) pairs

        Relations:
            - :class:`MultipleNegativesRankingLoss` is similar to this loss, but it does not use
              a guide model to guide the in-batch negative sample selection. `GISTEmbedLoss` yields
              a stronger training signal at the cost of some training overhead.

        Inputs:
            +---------------------------------------+--------+
            | Texts                                 | Labels |
            +=======================================+========+
            | (anchor, positive, negative) triplets | none   |
            +---------------------------------------+--------+
            | (anchor, positive) pairs              | none   |
            +---------------------------------------+--------+

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                guide = SentenceTransformer("all-MiniLM-L6-v2")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.GISTEmbedLoss(model, guide)

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()
        dimr   z_Both the training model and the guiding model must be based on the `transformers` architecture.N)superr	   __init__r   r   r   r   CosineSimilaritysimilarity_fct
isinstancer   
ValueError	tokenizervocabmax_seq_lengthmust_retokenize)selfr   r   r   	__class__ h/var/www/html/alpaca_bot/venv/lib/python3.10/site-packages/sentence_transformers/losses/GISTEmbedLoss.pyr      s   >zGISTEmbedLoss.__init__embed1embed2c                 C   s   |  |d|dS )N   r   )r   	unsqueeze)r   r!   r"   r   r   r    
sim_matrixV   s   zGISTEmbedLoss.sim_matrixsentence_featureslabelsc                    s   fdd|D }t  /  jr, fdd|D } fdd|D } fdd|D } fdd|D }W d    n1 s?w   Y  d }d }t|dkrW|\}}	|\}
}nt|dkrh|\}}	}|\}
}}n	td	t| ||	} ||} |	|	} |
|} |
|
} ||}| d
d}t j	 |||k< t j	 |||k< t j	 |||k< |||g}|d ur׈ ||} |
|}t j	 |||k< |
| t j|dd j }t |d |j}t ||S )Nc                       g | ]	}  |d  qS sentence_embedding)r   .0sentence_featurer   r   r    
<listcomp>Z   s    z)GISTEmbedLoss.forward.<locals>.<listcomp>c                    s"   g | ]} j jj|d  ddqS )	input_idsT)skip_special_tokens)r   r   batch_decoder+   r.   r   r    r/   ]       c                    s   g | ]} j |qS r   )r   tokenize)r,   	sentencesr.   r   r    r/   a   s    c                    s"   g | ]} fd d|  D qS )c                    s    i | ]\}}||  jjqS r   )tor   device)r,   keyvaluer.   r   r    
<dictcomp>c   s     z4GISTEmbedLoss.forward.<locals>.<listcomp>.<dictcomp>)itemsr+   r.   r   r    r/   b   r3   c                    r(   r)   )r   r+   r.   r   r    r/   g   s          z"Expected 2 or 3 embeddings, got {}r   r#   r   r   )torchno_gradr   lenr   formatr%   diagonalviewinfappendcatr   arangesizelongr6   r7   r   CrossEntropyLoss)r   r&   r'   
embeddingsdecodedguide_embeddingsnegativenegative_guideanchorpositiveanchor_guidepositive_guideap_simaa_simpp_simguided_ap_simguided_aa_simguided_pp_sim
guided_simscoresan_simguided_an_simr   r.   r    forwardY   sT   







zGISTEmbedLoss.forwardc                 C   s   | j | jdS )Nr   r   r_   r.   r   r   r    get_config_dict   s   zGISTEmbedLoss.get_config_dictc                 C   s   dS )Na	  
@misc{solatorio2024gistembed,
    title={GISTEmbed: Guided In-sample Selection of Training Negatives for Text Embedding Fine-tuning}, 
    author={Aivin V. Solatorio},
    year={2024},
    eprint={2402.16829},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}
r   r.   r   r   r    citation   s   zGISTEmbedLoss.citation)r
   )__name__
__module____qualname__r   floatr   r   r%   r   r   strr^   r   r`   propertyra   __classcell__r   r   r   r    r	   
   s     K"Er	   )typingr   r   r   r>   r   r   sentence_transformers.modelsr   )sentence_transformers.SentenceTransformerr   Moduler	   r   r   r   r    <module>   s    