o
sh' @ s d dl Z d dlmZ ddlmZ ddlmZ ddlmZ ddl m
Z
m Z ddl m
Z
mZ dd lmZmZ d
d lmZ e rGd dlZd dlmZ d Zd
ZdZdZdZ dddZ G dd dZ dS ) N)Image )create_causal_mask)_get_model_class)
AutoConfig) MODEL_FOR_PRETRAINING_MAPPING
MODEL_MAPPING)PROCESSOR_MAPPING_NAMES
AutoProcessor)TOKENIZER_MAPPING_NAMES
AutoTokenizer )is_torch_availablez[92mz[93mz[0mu ■u ⬚![]()
c
st jdkrdddddf jdkr%ddddddf ttdd D }d}g }tD ]6\ } | krL|sL }d f< |dkrp| ksZ d krp d krd d7 d| | f< d}q:ddurfd d
tD d fd dtD }
|dur|dk} t| d
d |
| d
dt d|d |} tj | dt
t t dt t t d}
|d |
d |d dt|
d }dur|d7 }|| g }tD ]3\}||f dkr|dd
tt|ttD q|tt|tt qtttt| }|D ] }|durO|d d d | d d | nd q4tD ]\\ t|}v rpt | t n|}d fddtD }d}durd fddtD }|| dt d d | d| qXd|S )z
Generates an attention matrix from a given attention mask.
Optionally applies a sliding window mask (e.g., for Gemma2/3) and
marks regions where image tokens occur based on the specified `img_token`.
r N c s s | ] }t t|V qd S N)lenrepr).0word r e/var/www/html/alpaca_bot/venv/lib/python3.10/site-packages/transformers/utils/attention_visualizer.py 8 s z6generate_attention_matrix_from_mask..r r
c s$ g | ] fd dt D qS )c s0 g | ]}d | krk rn ndnd qS )r r
r r j)isliding_windowr r
I s 0 zBgenerate_attention_matrix_from_mask...)range)r )nr )r r r I s $ z7generate_attention_matrix_from_mask.. c 3 sX | ]'} d |f rt t t nd |kr t t t n d |f r&tntV qdS r NYELLOW BLACK_SQUARERESETGREEN WHITE_SQUAREr )maskr r r K s
)
boundariesz: i == j (diagonal) z: token_type_idszAttention MatrixzSliding Window Maskc S s g | ]
}t | t qS r )r$ r& )r kr r r r i s z | c 3 sl | ]1}| v r |f r v r t t t n |kr(t t t n |f r0tntV qd S r r# r )r img_tokenr) r wordsr r r x s
c 3 sx | ]7}| v r"v r"d f d |f kr"t t t n |kr.t t t n | r6tntV qdS r" r# r )r r/ sliding_window_masktoken_type_bucketsr r0 r r r s ,
z:
)intndimr max enumerater jointorchwherecumsumboolarange bucketizer' r% r& r$ appendljustliststrrjustmapzipr )r0 r) r/ r token_type_idsimage_seq_lengthmax_word_length
first_img_idxoutputr- row_dummy
is_specialr, legendf_stringvertical_headeridxrow word_repr colored_word row_displaysliding_window_rowr ) r r/ r) r r r1 r2 r r0 r #generate_attention_matrix_from_mask( s~
,"
$
.
rV c @ s: e Zd ZdefddZd defddZd defdd Zd
S ) AttentionMaskVisualizer
model_namec C s t |}d| _t| drt| dd | _zt|t}W n t y- t|t
}Y nw |d u r:t d| d|| _ G dd d|t
j}|||| _| j|j || _|| _d S )Nr r z Model name z- is not supported for attention visualizationc @ s e Zd Zdd ZdS )z7AttentionMaskVisualizer.__init__.._ModelWrapperc S s$ t j| t dd| _|| _d S )Nr
)nnModule__init__Linear dummy_moduleconfig)selfr^ rX r r r r[ s
z@AttentionMaskVisualizer.__init__.._ModelWrapper.__init__N)__name__
__module__ __qualname__r[ r r r r
_ModelWrapper s rc )r from_pretrained image_tokenhasattrget_text_configgetattrr r r Exceptionr
ValueError
mapped_clsrY rZ modeltodtyperepo_idr^ )r_ rX r^ rk rc r r r r[ s"
z AttentionMaskVisualizer.__init__r. input_sentencec C s | j ||d d S )N)suffix)visualize_attention_mask)r_ rp rq r r r __call__ s z AttentionMaskVisualizer.__call__c C sV | j }i }d }| jjtv rkd}ttj|ddj}d}t j
| j |d}t |dr-|j
}n
|j|jgd }|r?|d|}||||d d
} |j|jgd | _
| d }
d | v r`| d |d < |j| d
d } n$| jjtv rt
| j } | |} | |d dd }
n
td|jj dd|j_| |
j\}
}tj|
||jjf| j j d}t |}t |j||
|d d}|d ur| }
n|
d d!|
d||}
dt"d| jj d| j# d }d}t$d| t$dd| jj d| j d | j#j% &t"| d | t$| t'| |
| j
t(| jd d |d |d }t$| t$| d S ) Nzchttps://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg?download=trueT)streamr+ )rG re r r pt)imagestextrq return_tensorsattention_maskrF input_ids)rx z Model type z) does not support attention visualizationeager)rn )r^ input_embedsry cache_positionpast_key_valuesr
z##z Attention visualization for z | r r3 z" Attention visualization for [1m:z[0m z r )r/ r rF rG ))rl r^
model_typer r openrequestsgetrawr
rd ro rf re tokenizerconvert_ids_to_tokensimage_token_idreplacer r tokenizerj _attn_implementationtrainshaper9 zeros hidden_sizern r= r r< unsqueezeexpandr rk printr` centerrV rh )r_ rp rq rl kwargsrG img processorre inputsry tokensr
batch_size
seq_lengthr| r} causal_masktop_bottom_border side_borderrN r r r rr s
z0AttentionMaskVisualizer.visualize_attention_maskN)r. )r` ra rb rB r[ rs rr r r r r rW s rW )r NNN) r PILr
masking_utilsr models.auto.auto_factoryr models.auto.configuration_autor models.auto.modeling_autor r models.auto.processing_autor r
models.auto.tokenization_autor r import_utilsr r9 torch.nnrY r' r$ r& r% r( rV rW r r r r s(
l