o
    sh(                     @   s   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZ d dlm	Z	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ d dlZd dlZd d	lmZmZ d d
lm Z! d dlm"Z# d dl$m%Z%m&Z& d dl'm(Z( d dlm)Z) d dl*m+Z+ d dlm,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2m3Z3 d dl4m5Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z<m=Z=m>Z> e> rd dl?m@Z@mAZAmBZB eCeDZEerd dlFmGZG d dlHmIZI d dlJmKZK G dd de,ZLg dZMg dZNd eeOef fd!d"ZPe	G d#d$ d$eZQd%d&d eOfd'd(ZRdS ))    N)Counterdefaultdict)copy)	dataclassfieldfields)Path)python_versionindent)TYPE_CHECKINGAnyDictListLiteralOptionalTupleUnion)CardData	ModelCard)dataset_info)
model_info)
EvalResulteval_results_to_model_index)	yaml_dump)nn)tqdm)TrainerCallback)CodeCarbonCallback)make_markdown_table)TrainerControlTrainerState__version__)Transformer)$SentenceTransformerTrainingArguments)fullnameis_accelerate_availableis_datasets_available)DatasetDatasetDictValue)SentenceEvaluator)SentenceTransformer)SentenceTransformerTrainerc                       s   e Zd Zdddeeef ddf fddZded	ed
e	ddddf
ddZ
ded	ed
e	ddddf
ddZded	ed
e	dddeeef ddfddZded	ed
e	dddeeef ddfddZ  ZS )ModelCardCallbacktrainerr.   default_args_dictreturnNc                    sX   t    || _|| _dd | jjjD }|r|d |jj_||jj_|jj	d d S )Nc                 S   s   g | ]	}t |tr|qS  )
isinstancer   ).0callbackr3   r3   ^/var/www/html/alpaca_bot/venv/lib/python3.10/site-packages/sentence_transformers/model_card.py
<listcomp>1   s    z.ModelCardCallback.__init__.<locals>.<listcomp>r   generated_from_trainer)
super__init__r0   r1   callback_handler	callbacksmodelmodel_card_datacode_carbon_callbackadd_tags)selfr0   r1   r=   	__class__r3   r7   r;   ,   s   

zModelCardCallback.__init__argsstatecontrolr>   r-   c                 K   s   ddl m}m}m} | jjr|j| jj|jjd|j_| jj	r.|j| jj	|jj
d|j_
t| jjtr>t| jj }	n| jjg}	d}
|
t|	k rq|	|
 }t||||frgt|drg|j|	vrg|	|j |
d7 }
|
t|	k sK|j|	 d S )Nr   )AdaptiveLayerLossMatryoshka2dLossMatryoshkaLosstrainevalloss   )sentence_transformers.lossesrH   rI   rJ   r0   train_datasetr?   extract_dataset_metadatatrain_datasetseval_dataseteval_datasetsr4   rM   dictlistvalueslenhasattrappend
set_losses)rB   rE   rF   rG   r>   kwargsrH   rI   rJ   lossesloss_idxrM   r3   r3   r7   on_init_end<   s0   


zModelCardCallback.on_init_endc                    sJ   h d |  } fdd| D |j_ fdd| D |j_d S )N>   do_evaldo_testdo_trainrun_name	hub_token	report_to
eval_delay
eval_steps
output_dir
save_stepslogging_dirlogging_stepssave_strategylogging_strategysave_total_limitgreater_is_betterpush_to_hub_tokensamples_per_labelshow_progress_barlogging_first_stepevaluation_strategymetric_for_best_modelc                    s   i | ]\}}| vr||qS r3   r3   r5   keyvalue)ignore_keysr3   r7   
<dictcomp>   s    z4ModelCardCallback.on_train_begin.<locals>.<dictcomp>c                    s6   i | ]\}}| vr|j v r|j | kr||qS r3   )r1   rv   ry   rB   r3   r7   rz      s
     )to_dictitemsr?   all_hyperparametersnon_default_hyperparameters)rB   rE   rF   rG   r>   r\   	args_dictr3   r{   r7   on_train_begind   s   	

z ModelCardCallback.on_train_beginmetricsc                    sf    fdd D }|j jr#|j jd d |jkr#|j jd | d S |j j|j|jd| d S )Nc                    s4   i | ]}| d rd|ddd  | qS )_loss _rN   N)endswithjoinsplitr5   rw   r   r3   r7   rz      s   4 z1ModelCardCallback.on_evaluate.<locals>.<dictcomp>StepEpochr   )r?   training_logsglobal_stepupdaterZ   epoch)rB   rE   rF   rG   r>   r   r\   	loss_dictr3   r   r7   on_evaluate   s   	zModelCardCallback.on_evaluatelogsc                 K   sv   dht |@ }|r9|jjr&|jjd d |jkr&||  |jjd d< d S |jj|j|j||  d d S d S )NrM   r   r   Training Loss)r   r   r   )setr?   r   r   poprZ   r   )rB   rE   rF   rG   r>   r   r\   keysr3   r3   r7   on_log   s   	
zModelCardCallback.on_log)__name__
__module____qualname__r   strr   r;   r%   r!   r    r_   r   floatr   r   __classcell__r3   r3   rC   r7   r/   +   sd    "
(
+


r/   )languagelicenselibrary_nametagsdatasetsr   pipeline_tagwidgetmodel-indexco2_eq_emissions
base_model)r>   r0   eval_results_dictr2   c                  C   s`   t  ttjtjd} t rddlm} || d< t r$ddlm} || d< ddl	m} || d< | S )N)pythonsentence_transformerstransformerstorchr   r"   
accelerater   
tokenizers)
r	   sentence_transformers_versionr   r#   r   r'   r   r(   r   r   )versionsaccelerate_versiondatasets_versiontokenizers_versionr3   r3   r7   get_versions   s   r   c                
   @   s  e Zd ZU dZeedZeee	e
e	 f  ed< dZee	 ed< dZee	 ed< dZee	 ed< eedZe
ee	e	f  ed< eedZe
ee	e	f  ed	< d
Ze	ed< edd dZee
e	  ed< dZed ed< edddZee	 ed< edddZee	 ed< eeddZee	ef ed< eeddZee	ef ed< eeddZeedee	ef f  ed< eeddZe
ee	ef  ed< eeddZe
ee	e	f  ed< edddZ ee	 ed< eeddZ!e
ee	e	f  ed< edddZ"ee# ed< eeddZ$ee	e	f ed< edddZ%ee& ed < edddd!Z'ed" ed#< eeddd$Z(e
e	 ed%< ed&ddZ)e*ed'< ed(ddZ+e&ed)< ed*ddZ,e	ed+< ed,ddZ-e	ed-< ee.ddZ/ee	e	f ed.< edddd!Z0ed/ ed0< dmd2d3Z1dnd4e*d1dfd5d6Z2d7e
e3j4 d1dfd8d9Z5d:e&d1dfd;d<Z6d=ed> d1dfd?d@Z7dAddBee	ef d1dfdCdDZ8dodFdGZ9	dpd=ed> dHee	 d1e
ee	e	f  fdIdJZ:d=ee	e	f dKee	ef dLeeee	e3j4f e3j4f  d1ee	e	f fdMdNZ;d=ed> dOedP d1ee	ef fdQdRZ<dqdSdTZ=de	d1dfdUdVZ>dpde	dWee	 d1dfdXdYZ?dee	e
e	 f d1dfdZd[Z@de	d1dfd\d]ZAdee	e
e	 f d1dfd^d_ZBdmd`daZCd1ee	ef fdbdcZDddde ZEd1eedf ee	ef f fdgdhZFd1ee	ef fdidjZGdpd1e	fdkdlZHdS )r SentenceTransformerModelCardDataa  A dataclass storing data used in the model card.

    Args:
        language (`Optional[Union[str, List[str]]]`): The model language, either a string or a list,
            e.g. "en" or ["en", "de", "nl"]
        license (`Optional[str]`): The license of the model, e.g. "apache-2.0", "mit",
            or "cc-by-nc-sa-4.0"
        model_name (`Optional[str]`): The pretty name of the model, e.g. "SentenceTransformer based on microsoft/mpnet-base".
        model_id (`Optional[str]`): The model ID when pushing the model to the Hub,
            e.g. "tomaarsen/sbert-mpnet-base-allnli".
        train_datasets (`List[Dict[str, str]]`): A list of the names and/or Hugging Face dataset IDs of the training datasets.
            e.g. [{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}, {"name": "STSB"}]
        eval_datasets (`List[Dict[str, str]]`): A list of the names and/or Hugging Face dataset IDs of the evaluation datasets.
            e.g. [{"name": "SNLI", "id": "stanfordnlp/snli"}, {"id": "mteb/stsbenchmark-sts"}]
        task_name (`str`): The human-readable task the model is trained on,
            e.g. "semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more".
        tags (`Optional[List[str]]`): A list of tags for the model,
            e.g. ["sentence-transformers", "sentence-similarity", "feature-extraction"].

    .. tip::

        Install `codecarbon <https://github.com/mlco2/codecarbon>`_ to automatically track carbon emission usage and
        include it in your model cards.

    Example::

        >>> model = SentenceTransformer(
        ...     "microsoft/mpnet-base",
        ...     model_card_data=SentenceTransformerModelCardData(
        ...         model_id="tomaarsen/sbert-mpnet-base-allnli",
        ...         train_datasets=[{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}],
        ...         eval_datasets=[{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}],
        ...         license="apache-2.0",
        ...         language="en",
        ...     ),
        ... )
    )default_factoryr   Nr   
model_namemodel_idrR   rT   zjsemantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more	task_namec                   C   s   g dS )N)sentence-transformerssentence-similarityzfeature-extractionr3   r3   r3   r3   r7   <lambda>      z)SentenceTransformerModelCardData.<lambda>r   
deprecatedgenerate_widget_examplesF)defaultinitr   base_model_revision)r   r   r   r~   r,   r   r   r   predict_examplelabel_example_listr@   	citationsbest_model_step)r   r   reprr.   r0   )r   r   r   r   T
first_saver   widget_stepr   r   r   r   versionr-   r>   r2   c                 C   s~   | j  }t| j tr| j g| _ | j| j|d| _| j| j|d| _| jr;| jddkr=t	d| jd d | _d S d S d S )N)infer_languages/rN   zThe provided z} model ID should include the organization or user, such as "tomaarsen/mpnet-base-nli-matryoshka". Setting `model_id` to None.)
r   r4   r   validate_datasetsrR   rT   r   countloggerwarning)rB   r   r3   r3   r7   __post_init__>  s   

z.SentenceTransformerModelCardData.__post_init__r   c              	   C   s   g }|D ]u}d|vrd|v r|d |d< d|v rtzt |d }W n ty7   td|d d |d= Y n=w |jrg|rgd|jv rg|jd}|d u rO |S t|trW|g}|D ]}|| jvrf| j	| qY|j
| jvrt| j	|j
 |	| q|S )NnameidzThe dataset `id` z5 does not exist on the Hub. Setting the `id` to None.r   )get_dataset_info	Exceptionr   r   cardDatagetr4   r   r   rZ   r   r   )rB   dataset_listr   output_dataset_listdatasetinfodataset_languager   r3   r3   r7   r   N  s:   


z2SentenceTransformerModelCardData.validate_datasetsr]   c              	      s   ddi}|D ]}z	|j ||jj< W q ty   Y qw tt}| D ]\}}|| | q#dtt	 dt	fdd  fdd| D | _
| d	d
 dd |D D  d S )NzSentence Transformersa  
@inproceedings{reimers-2019-sentence-bert,
    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
    author = "Reimers, Nils and Gurevych, Iryna",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
    month = "11",
    year = "2019",
    publisher = "Association for Computational Linguistics",
    url = "https://arxiv.org/abs/1908.10084",
}
r]   r2   c                 S   s2   t | dkrd| d d d | d  S | d S )NrN   z, r   z and r   )rX   r   )r]   r3   r3   r7   	join_list  s   z>SentenceTransformerModelCardData.set_losses.<locals>.join_listc                    s   i | ]	\}} ||qS r3   r3   )r5   citationr]   r   r3   r7   rz         z?SentenceTransformerModelCardData.set_losses.<locals>.<dictcomp>c                 S   s   g | ]}d | qS )zloss:r3   r5   rM   r3   r3   r7   r8         z?SentenceTransformerModelCardData.set_losses.<locals>.<listcomp>c                 S   s   i | ]}|j j|qS r3   )rD   r   r   r3   r3   r7   rz     r   )r   rD   r   r   r   rV   r}   rZ   r   r   r   rA   )rB   r]   r   rM   inverted_citationsr   r3   r   r7   r[   p  s   "z+SentenceTransformerModelCardData.set_lossesstepc                 C   
   || _ d S N)r   )rB   r   r3   r3   r7   set_best_model_step     
z4SentenceTransformerModelCardData.set_best_model_stepr   )r)   r*   c              	   C   s  t |tr
t|d}g | _ttjt| dd}d}t	|
 ddddD ]\}}d	d
 || j
 D }|| |}t|}i }	t|tjt|t||dD ]\}
}tdd | D |	|
< qTtt|	
 dd d \}}|d | t||d  d d d }}|D ]_}
t||
  }t|dk r|r| }t||  }t|dkr|| n||d  t|dk r|st|dk rq| j|d tj|dd  t|d dd |d d | _qq&d S )N)r      )k  zComputing widget examplesexampleF)descunitleavec                 S   s0   g | ]\}}t |tr|jd kr|dkr|qS )stringdataset_name)r4   r+   dtype)r5   columnfeaturer3   r3   r7   r8     s    zHSentenceTransformerModelCardData.set_widget_examples.<locals>.<listcomp>c                 s   s    | ]}t |V  qd S r   rX   )r5   rx   r3   r3   r7   	<genexpr>  s    zGSentenceTransformerModelCardData.set_widget_examples.<locals>.<genexpr>c                 S   s   | d S )NrN   r3   )xr3   r3   r7   r     r   zFSentenceTransformerModelCardData.set_widget_examples.<locals>.<lambda>rw   r      rN   r   )source_sentence	sentences   )r4   r)   r*   r   r   randomchoicesrV   r   r   r}   featuresselect_columnsrX   	enumerateselectsamplerangeminsumrW   zipsortedr   extendrZ   r   )rB   r   dataset_namesnum_samples_to_checkr   num_samplescolumnsstr_datasetdataset_sizelengthsidxr  indicesr   target_indicesbackup_indicesr   
backup_idxbackup_sampler3   r3   r7   set_widget_examples  sJ   

(
&z4SentenceTransformerModelCardData.set_widget_examples	evaluatorr   c                    s   ddl m} t|| j|< t|dro|j  rqt||r%dd |jD  nt tr- g | j	d u r7d}d}n
| j	j
j}| j	j
j} fdd| D }| jrb| jd d	 |krb| jd | d S | j||d
| d S d S d S )Nr   )SequentialEvaluatorprimary_metricc                 S   s   g | ]}|j qS r3   r  )r5   sub_evaluatorr3   r3   r7   r8     s    zKSentenceTransformerModelCardData.set_evaluation_metrics.<locals>.<listcomp>c                    s   i | ]\}}| v r||qS r3   r3   rv   primary_metricsr3   r7   rz         zKSentenceTransformerModelCardData.set_evaluation_metrics.<locals>.<dictcomp>r   r   r   ) sentence_transformers.evaluationr  r   r   rY   r  r4   
evaluatorsr   r0   rF   r   r   r}   r   r   rZ   )rB   r  r   r  r   r   training_log_metricsr3   r   r7   set_evaluation_metrics  s.   




z7SentenceTransformerModelCardData.set_evaluation_metricsr)   c                    s   d}t t}t }|D ]1}|d }|d }||vr3|| dt| d t|| |kr3|| t| jkr< nq fdd| D  _	d S )Nr   textlabelz<li>z</li>c                    sF   g | ]\}} j jrt|tr j j| n|d d| d dqS )z<ul> z</ul>)LabelExamples)r>   labelsr4   intr   )r5   r(  example_setrB   r3   r7   r8     s     zGSentenceTransformerModelCardData.set_label_examples.<locals>.<listcomp>)
r   rV   r   rZ   r   rX   addnum_classesr}   r   )rB   r   num_examples_per_labelexamplesfinished_labelsr  r'  r(  r3   r/  r7   set_label_examples  s    

z3SentenceTransformerModelCardData.set_label_examplesr   c                    s0  t |tr fdd| D S dtt dtt dtfdd}|j}i }|r.td|r.d }|r4||d	< |rd
|d v rt	|d d
 j
}d}|||}|dkrS|gS ||t|  }	d|	v rp|	d\}
}|
 d| |d< nd }
|	}z	t|j|d< W n	 ty   Y nw t|d dkr|d |d< |gS )Nc                    s(   g | ]\}} j ||d D ]}|qqS ))r   )infer_datasets)r5   r   sub_datasetr   r/  r3   r7   r8     s    zCSentenceTransformerModelCardData.infer_datasets.<locals>.<listcomp>tuplesubtupler2   c                 S   sB   t | D ]\}}||d kr| ||t|  |kr|  S qdS )Nr   r   )r  rX   )r8  r9  ielementr3   r3   r7   subtuple_finder  s
   $zHSentenceTransformerModelCardData.infer_datasets.<locals>.subtuple_finderz_dataset_\d+r   filenamer   )huggingfacer   r   ___r   r   (   revision)r4   r*   r}   r   r   r-  cache_filesrematchr   partsrX   r   r   r   r   )rB   r   r   r<  rC  dataset_outputcache_path_partsr9  indexcache_dataset_nameauthorr3   r/  r7   r6    s>   


z/SentenceTransformerModelCardData.infer_datasetsr   rM   c                    s  |si S t ||d< dd |jD |d< i |d< |jD ]}|dd | d }t|tr| j}t|trKd	|v rK|d	 jd
d }d}n	dd D }d}dt	t
|d d| t	t|t | d d| t	t|d d| dd|d |< qt|ttfrt d fddt D d|d |< qt|trdt	t
|| dt	t|| t | dt	t|| ddd|d |< qt|trtdd D  t  d
krddt | did|d |< qdt
  dt t   ddt  ddd|d |< qt|i d|d |< qdtfdddd id!d |d  D dd"ifd#d|d  D g}	tt|	d$d%d&|d'< |dd( |d)< t |d) t|d) d  }
g }t|
D ]D}i }|jD ]6}|d) | | }t|trt |d*krt|dd* dd+ d, }t|d-d.}d/| d0||< qz|| qstt|d$d%d&|d1< d2t|i|d3< t|d4r| }z	tj|d5d6}W n ty   t|}Y nw td7| d8d&|d3 d9< |S ):a  
        Given a dataset, compute the following:
        * Dataset Size
        * Dataset Columns
        * Dataset Stats
            - Strings: min, mean, max word count/token length
            - Integers: Counter() instance
            - Floats: min, mean, max range
            - List: number of elements or min, mean, max number of elements
        * 3 Example samples
        * Loss function name
            - Loss function config
        sizec                 S   s   g | ]}d | dqS )<code></code>r3   )r5   r   r3   r3   r7   r8   F      zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<listcomp>r  statsNr   r   attention_maskrN   )dimtokensc                 S      g | ]}t |qS r3   r   )r5   sentencer3   r3   r7   r8   Q      
charactersr      r   )r  meanmax)r   datar-  c                    s8   i | ]}|t  d krdnd  | t  dqS )rN   ~r)  z.2%r   r   )counter
subsectionr3   r7   rz   _  s    *zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<dictcomp>r   c                 S   rT  r3   r   )r5   lstr3   r3   r7   r8   n  rV  rV   z	 elementsz.2fr[  c                 S   s    dd dd |  D  d S )Nz<ul><li>z	</li><li>c                 s   s"    | ]\}}| d | V  qdS )z: Nr3   rv   r3   r3   r7   r     s     zaSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.to_html_list.<locals>.<genexpr>z
</li></ul>)r   r}   r[  r3   r3   r7   to_html_list  s    zNSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.to_html_listr)  typec                 S   s   i | ]	\}}||d  qS )r   r3   rv   r3   r3   r7   rz     r   detailsc                    s   i | ]\}}| |d  qS r`  r3   rv   )ra  r3   r7   rz     r"  -:|--|z  stats_tabler   r3  r   r   z, ...]
z<br>rM  rN  examples_tabler&   rM   get_config_dictr   r
   z```json
z
```config_code)rX   column_namesr4   r   r>   tokenizerU   r	  tolistroundr  rZ  r-  boolr   r  r   rV   r&   r}   r   r   replacer  rZ   rY   ri  jsondumps	TypeError)rB   r   r   rM   r   first	tokenizedr  suffixstats_linesr  examples_lines
sample_idxr  rx   config
str_configr3   )r]  r^  ra  r7   compute_dataset_metrics/  s   

	 
z8SentenceTransformerModelCardData.compute_dataset_metricsdataset_type)rK   rL   c              	      s   |rW|r-t |trt|t|kst |tr-t|dkr-td| d| d| d g }|s4 |}t |trK fddt| |	 |D }n 
||d  jjg}|d	krntd
d |D }|rn d|   |S )NrN   zThe number of `z?_datasets` in the model card data does not match the number of z1 datasets in the Trainer. Removing the provided `z$_datasets` from the model card data.c              	      s>   g | ]\}}}  ||t jjtr jj| n jjqS r3   )r|  r4   r0   rM   rU   )r5   r   dataset_valuer   r/  r3   r7   r8     s     zMSentenceTransformerModelCardData.extract_dataset_metadata.<locals>.<listcomp>r   rK   c                 S   s   g | ]}| d dqS )rL  r   )r   )r5   metadatar3   r3   r7   r8     rO  zdataset_size:)r4   r*   rX   r)   r   r   r6  r
  r   rW   r|  r0   rM   r	  rA   r   )rB   r   dataset_metadatar}  num_training_samplesr3   r/  r7   rQ     s8   



z9SentenceTransformerModelCardData.extract_dataset_metadatac                 C   r   r   )r>   )rB   r>   r3   r3   r7   register_model  r   z/SentenceTransformerModelCardData.register_modelc                 C   r   r   )r   )rB   r   r3   r3   r7   set_model_id  r   z-SentenceTransformerModelCardData.set_model_idrB  c                 C   sJ   zt |}W n
 ty   Y dS w |j| _|d u s|dkr |j}|| _dS )NFmainT)get_model_infor   r   r   shar   )rB   r   rB  r   r3   r3   r7   set_base_model  s   z/SentenceTransformerModelCardData.set_base_modelc                 C   s   t |tr|g}|| _d S r   )r4   r   r   )rB   r   r3   r3   r7   set_language  s   

z-SentenceTransformerModelCardData.set_languagec                 C   r   r   )r   )rB   r   r3   r3   r7   set_license  r   z,SentenceTransformerModelCardData.set_licensec                 C   s4   t |tr|g}|D ]}|| jvr| j| q
d S r   )r4   r   r   rZ   )rB   r   tagr3   r3   r7   rA     s   

z)SentenceTransformerModelCardData.add_tagsc                    s   t | jd trB| jd jjj}t|}d|jdd  g}|j	
d | fddtdt D 7 }|D ]}| |rA d S q7d S d S )Nr   r   r@  r   c                    s4   g | ]}d   d| d d   |d  qS )r   Nr   )r   )r5   r  splitsr3   r7   r8     s    (zJSentenceTransformerModelCardData.try_to_set_base_model.<locals>.<listcomp>rN   )r4   r>   r$   
auto_modelrz  _name_or_pathr   r   rF  r   r   r  rX   r  )rB   r   base_model_pathcandidate_model_idsr   r3   r  r7   try_to_set_base_model  s   
z6SentenceTransformerModelCardData.try_to_set_base_modelc              
      sJ  g }i }g }| j  D ]\}}t|ddt|ddrHtfdd| D rHfdd| D }rHd rHtd	 d d
tdtfddfdd| D }fdd| D }|jt|dd |	t
| t|ddd | fdd| D  || q|t| t| j|dS )au  Format the evaluation metrics for the model card.

        The following keys will be returned:
        - eval_metrics: A list of dictionaries containing the class name, description, dataset name, and a markdown table
          This is used to display the evaluation metrics in the model card.
        - metrics: A list of all metric keys. This is used in the model card metadata.
        - model-index: A list of dictionaries containing the task name, task type, dataset type, dataset name, metric name,
          metric type, and metric value. This is used to display the evaluation metrics in the model card metadata.
        r   Nr  c                 3   s    | ]
}|  d  V  qdS )r   N)
startswithr   r   r3   r7   r     s    zGSentenceTransformerModelCardData.format_eval_metrics.<locals>.<genexpr>c                    s&   i | ]\}}|t  d  d |qS )rN   Nr   rv   r  r3   r7   rz        & zHSentenceTransformerModelCardData.format_eval_metrics.<locals>.<dictcomp>r   rN   rx   r2   c                 S   s0   zt | dr|  W S W | S  ty   Y | S w )z^Try to convert a value from a Numpy or Torch scalar to pure Python, if not already pure Pythonr   )rY   itemr   )rx   r3   r3   r7   try_to_pure_python  s   

zPSentenceTransformerModelCardData.format_eval_metrics.<locals>.try_to_pure_pythonc                    s   i | ]	\}}| |qS r3   r3   rv   )r  r3   r7   rz      r   c                    sN   g | ]#\}}| krd | d n|| krd t |d d nt |ddqS )**r   )Metricr+   )rn  r5   
metric_keymetric_valuer  r3   r7   r8   "  s    zHSentenceTransformerModelCardData.format_eval_metrics.<locals>.<listcomp>rd  re  )
class_namedescriptionr   tablec                    sd   g | ].\}}t |ttfrt d d pd dd dd p$d|dd  ||dqS )r   -unknownr   Unknown)r   	task_typer}  r   metric_namemetric_typer  )r4   r-  r   r   lowerrp  titler  )r   r  r3   r7   r8   8  s    
)eval_metricsr   r   )r   r}   getattrallr   r  rX   r   r  rZ   r&   r   rp  r  r   rV   r   r   )rB   r  all_metricseval_resultsr  r   table_linesr3   )r   r  r   r  r  r7   format_eval_metrics  sD   
	



z4SentenceTransformerModelCardData.format_eval_metricsc                    sZ   dd  j D }dtdtfdd}t||d fdd	 j D }t|}|d
|v dS )Nc                 S   s   h | ]}|  D ]}|qqS r3   )r   )r5   linesrw   r3   r3   r7   	<setcomp>P  r"  zHSentenceTransformerModelCardData.format_training_logs.<locals>.<setcomp>rw   r2   c                 S   s6   | dkrdS | dkrdS | dkrdS |  drdS | S )	Nr   0r   1r   2rM   3)r   r   r3   r3   r7   sort_metricsS  s   
zKSentenceTransformerModelCardData.format_training_logs.<locals>.sort_metricsr   c                    s    g | ]  fd dD qS )c                    sJ   i | ]!}| d  j krd| v rt | dnd dn |dqS )r   r  r   r  )r   rn  r   r   )linerB   r3   r7   rz   `  s    "
zTSentenceTransformerModelCardData.format_training_logs.<locals>.<listcomp>.<dictcomp>r3   )r5   rB   sorted_eval_lines_keys)r  r7   r8   _  s    zISentenceTransformerModelCardData.format_training_logs.<locals>.<listcomp>r  )
eval_linesexplain_bold_in_eval)r   r   r  r   )rB   eval_lines_keysr  r   r  r3   r  r7   format_training_logsN  s   	z5SentenceTransformerModelCardData.format_training_logsr   c                 C   sd   | j j }dt|jd t|jdd|jdk|j|jt	|j
d ddi}|jr0|j|d d	< |S )
Nr   r   
codecarbonzfine-tuningYi  r   )	emissionsenergy_consumedsourcetraining_typeon_cloud	cpu_modelram_total_size
hours_usedhardware_used)r@   tracker_prepare_emissions_datar   r  r  r  r  r  rn  duration	gpu_model)rB   emissions_dataresultsr3   r3   r7   get_codecarbon_datan  s   z4SentenceTransformerModelCardData.get_codecarbon_datac              
      s   j s jd ur jjp jj }r |  jr- js-z   W n	 ty,   Y nw  j	s> jr;d j  _	nd _	 fddt
 D } jrlz	|   W n tyk } z
td|  |d }~ww  jrz	|   W n ty } ztd|  W Y d }~nd }~ww t jdk|d<  jr jjr jjjd ur|    j |d	<  j |d
< t j|d<  jjrddddd jj jjdd |d< nd|d< d _tD ]}| |d  q|S )NzSentenceTransformer based on r-   c                    s   i | ]
}|j t |j qS r3   )r   r  )r5   r   r/  r3   r7   rz     s    z<SentenceTransformerModelCardData.to_dict.<locals>.<dictcomp>z+Error while formatting evaluation metrics: z&Error while formatting training logs: d   hide_eval_linesmodel_max_lengthoutput_dimensionalitymodel_stringzCosine SimilarityzDot ProductzEuclidean DistancezManhattan Distance)cosinedot	euclidean	manhattanr   r   similarity_fn_nameF)!r   r0   rS   rP   r  r   r   r  r   r   r   r   r   r  r   r   r   r  rX   r@   r  _start_timer  r>   get_max_seq_length get_sentence_embedding_dimensionr   r  r   rp  r  IGNORED_FIELDSr   )rB   r   
super_dictexcrw   r3   r/  r7   r|     sn   

z(SentenceTransformerModelCardData.to_dictc                 C   s$   t dd |   D d|d S )Nc                 S   s&   i | ]\}}|t v r|d ur||qS r   )YAML_FIELDSrv   r3   r3   r7   rz     r  z<SentenceTransformerModelCardData.to_yaml.<locals>.<dictcomp>F)	sort_keys
line_break)r   r|   r}   strip)rB   r  r3   r3   r7   to_yaml  s   z(SentenceTransformerModelCardData.to_yaml)r2   N)T)r   r)   r2   Nr   )r>   r-   r2   N)Ir   r   r   __doc__r   rV   r   r   r   r   r   __annotations__r   r   r   rR   r   rT   r   r   r   r   r   r   rU   r   r   r~   r   r   r   r   r   r   r@   r   r   r   r-  r0   r   r   ro  r   r   r   r   r   r>   r   r   r   Moduler[   r   r  r&  r5  r6  r|  rQ   r  r  r  r  r  rA   r  r  r  r  r|   r  r3   r3   r3   r7   r      s   
 "'
(   
"3

6



x


'
L" Fr   r>   r-   c                 C   s&   t tjd }tj| j|dd}|jS )Nzmodel_card_template.mdu   🤗)	card_datatemplate_pathhf_emoji)r   __file__parentr   from_templater?   content)r>   r  
model_cardr3   r3   r7   generate_model_card  s   r  )Srq  loggingr   rD  collectionsr   r   r   dataclassesr   r   r   pathlibr   platformr	   textwrapr   typingr   r   r   r   r   r   r   r   r   r   huggingface_hubr   r   r   r   r   r  huggingface_hub.repocard_datar   r   huggingface_hub.utilsr   r   tqdm.autonotebookr   r   transformers.integrationsr   transformers.modelcardr   transformers.trainer_callbackr    r!   r   r#   r   sentence_transformers.modelsr$   #sentence_transformers.training_argsr%   sentence_transformers.utilr&   r'   r(   r   r)   r*   r+   	getLoggerr   r   2sentence_transformers.evaluation.SentenceEvaluatorr,   )sentence_transformers.SentenceTransformerr-   sentence_transformers.trainerr.   r/   r  r  r   r   r   r  r3   r3   r3   r7   <module>   s`    (
      l