§
    ‚Štj3"  ã                   óÄ  — d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
Z
d dlmZ d dlmZ dd	lmZmZmZ  G d
„ d¦  «        Ze G d„ d¦  «        ¦   «         Zdedefd„Zdedededefd„Zd+dedededefd„Zdedededefd„Z	 d,de
j        dee         dee         deddf
d „Zd!ed"ed#ed$ed%e	dee         fd&„Zd'e j        dee         fd(„Z d)„ Z!ed*„ ¦   «         Z"dS )-é    N)Úcontextmanager)Ú	dataclass)ÚceilÚlog2)ÚAny)ÚPretrainedConfig)Úis_torch_greater_or_equalé   )ÚFutureRequestStateÚRequestStateÚRequestStatusc                   ó”   — e Zd ZdZdd„Zdd„Zdeedf         dej	        j
        dz  fd„Zdeedf         d	ej	        j
        ddfd
„ZdS )ÚCudaGraphBufferz`A dict for CUDA graphs with a special __del__ method to make sure the graphs are properly reset.ÚreturnNc                 ó   — i | _         d S ©N©Ú_storage)Úselfs    úo/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/generation/continuous_batching/utils.pyÚ__init__zCudaGraphBuffer.__init__   s   € ØEGˆŒˆˆó    c                 ó†   — | j         r9| j                              ¦   «         \  }}|                     ¦   «          | j         °7d S d S r   )r   ÚpopitemÚreset)r   Ú_Úgraphs      r   Ú__del__zCudaGraphBuffer.__del__"   sP   € ØŒmð 	Ø”}×,Ò,Ñ.Ô.‰HˆAˆuØ�KŠK‰MŒMˆMð Œmð 	ð 	ð 	ð 	ð 	r   Úkey.c                 ó6   — | j                              |¦  «        S r   )r   Úget)r   r   s     r   Ú	get_graphzCudaGraphBuffer.get_graph'   s   € ØŒ}× Ò  Ñ%Ô%Ð%r   r   c                 ó   — || j         |<   d S r   r   )r   r   r   s      r   Ú	set_graphzCudaGraphBuffer.set_graph*   s   € Ø"ˆŒ�cÑÐÐr   )r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtupleÚintÚtorchÚcudaÚ	CUDAGraphr"   r$   © r   r   r   r      s°   € € € € € ØjÐjðHð Hð Hð Hðð ð ð ð
&˜U 3¨ 8œ_ð &°´Ô1EÈÑ1Lð &ð &ð &ð &ð#˜U 3¨ 8œ_ð #°U´ZÔ5Ið #Èdð #ð #ð #ð #ð #ð #r   r   c                   ó>   — e Zd ZU dZdZeed<   dZeed<   dZeed<   dS )ÚWorkloadHintszRA tiny dataclass containing hints to help choose good continuous batching defaultsr   Úmax_prompt_lengthÚmax_generated_lengthÚnum_requestsN)	r%   r&   r'   r(   r1   r*   Ú__annotations__r2   r3   r.   r   r   r0   r0   .   sJ   € € € € € € à\Ð\àÐ�sÐÐÑØ !Ð˜#Ð!Ð!Ñ!Ø€L�#ÐÐÑÐÐr   r0   Úconfigr   c                 ó   — | j         dv S )z:Checks if attention mask is needed for the given (config).)zpaged|eagerz
paged|sdpa)Ú_attn_implementation)r5   s    r   Úattn_mask_is_neededr8   7   s   € àÔ&Ð*GÐGÐGr   ÚsizeÚinterval_sizeÚ	max_valuec                 ól   — |dk    r|S | dk    rt          | |z  ¦  «        |z  n|}t          ||¦  «        S )zQReturn the smallest multiple of (interval_size) >= (size), capped at (max_value).r   )r   Úmin)r9   r:   r;   Úpaddeds       r   Úpad_to_intervalr?   <   sG   € à˜ÒÐØÐØ;?À!º8¸8�T�$˜Ñ&Ñ'Ô'¨-Ñ7Ð7È€FÝˆv�yÑ!Ô!Ð!r   ÚvalueÚ	min_valuec                 ó¶   — t          | t          d|¦  «        ¦  «        } dt          t          t          | ¦  «        ¦  «        ¦  «        z  }t	          ||¦  «        S )z‘Return the smallest power of 2 >= (value), capped at (max_value). If a minimum value is provided, the value is at
    least padded to that value.r
   é   )Úmaxr*   r   r   r=   )r@   r;   rA   r>   s       r   Úpad_to_pow2rE   D   sM   € õ �•s˜1˜iÑ(Ô(Ñ)Ô)€EØ•#•d�4 ™;œ;Ñ'Ô'Ñ(Ô(Ñ(€FÝˆv�yÑ!Ô!Ð!r   ÚxÚ	divide_byÚalign_toc                 ód   — t          t          | |z  ¦  «        ¦  «        } | |z  r| || |z  z
  z  } | S r   )r*   r   )rF   rG   rH   s      r   Úaligned_dividerJ   L   s?   € Ý�D��Y‘ÑÔÑ Ô €AØˆ8�|ð 'Ø	ˆX˜˜X™Ñ&Ñ&ˆØ€Hr   Úattention_maskÚcumulative_seqlens_qÚcumulative_seqlens_kÚsliding_windowc                 ój  — t          j        | j        ¦  «        j        }t	          t          |¦  «        dz
  ¦  «        D ]ô}||dz            ||         z
  }||dz            ||         z
  }||k     r|dk    r	||z
  dz   }nd}t          ||         ||dz            ¦  «        }	t          ||         ||dz            ¦  «        }
t          j        | d|	|
f         j        || j        | j	        ¬¦  «        }t          j
        ||¬¦  «        }|dk    r!||z
  |z
  }|t          j        ||¬¦  «        z  }|| d|	|
f<   ŒõdS )u  Builds an attention mask inplace using the cumulative seqlens of the query and key. If given a sliding window, it
    will also apply a sliding window mask on top. The attention mask is not boolean, it uses zeroes and -inf (or its
    equivalent) so it's more of an attention score bias tensor.
    The attention mask is a block-diagonal matrix, with each block an attention mask for a single query-key pair.
    Each of those block is built from a causal mask and, if there is a sliding window, a sliding window mask.

    An example is represented below, with seqlen_k = 8, seqlen_q = 4 and sliding_window = 6:

    CAUSAL MASK:

           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    SLIDING WINDOW MASK:
         â”Œâ”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€ seqlen_k - seqlen_q - sliding_window = 8 - 4 - 6 = -2 offset to the left
       <â”€â”´â”€>
     â–‘ â–ˆ | â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–‘ â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    ATTENTION MASK (sum of causal and sliding window masks):

           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–‘ â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    Another example with seqlen_k = 5, seqlen_q = 3 and sliding_window = 2:

    CAUSAL MASK:

           â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    SLIDING WINDOW MASK:
         â”Œâ”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€ seqlen_k - seqlen_q - sliding_window = 5 - 3 - 2 = 0 offset to the left
        <â”´>
         | â–‘ â–ˆ â–ˆ â–ˆ â–ˆ
         | â–‘ â–‘ â–ˆ â–ˆ â–ˆ
         | â–‘ â–‘ â–‘ â–ˆ â–ˆ

    ATTENTION MASK (sum of causal and sliding window masks):

           â–‘ â–ˆ â–ˆ â–‘ â–‘
           â–‘ â–‘ â–ˆ â–ˆ â–‘
           â–‘ â–‘ â–‘ â–ˆ â–ˆ

    r
   .)ÚdtypeÚdevice)ÚdiagonalN)r+   ÚfinforP   r=   ÚrangeÚlenÚsliceÚfullÚshaperQ   ÚtriuÚtril)rK   rL   rM   rN   rA   ÚiÚseqlen_qÚseqlen_kÚcausal_diagonalÚquery_rangeÚ	key_rangeÚ	minus_infÚmaskedÚsliding_diagonals                 r   Úbuild_attention_maskrd   S   sx  € õt ”˜NÔ0Ñ1Ô1Ô5€IÝ•3Ð+Ñ,Ô,¨qÑ0Ñ1Ô1ð =ð =ˆØ'¨¨A©Ô.Ð1EÀaÔ1HÑHˆØ'¨¨A©Ô.Ð1EÀaÔ1HÑHˆØ�hÒÐ 8¨q¢= =Ø&¨Ñ1°AÑ5ˆOˆOàˆOÝÐ0°Ô3Ð5IÈ!ÈaÉ%Ô5PÑQÔQˆÝÐ.¨qÔ1Ð3GÈÈAÉÔ3NÑOÔOˆ	å”JØ˜3 ¨YÐ6Ô7Ô=ØØ Ô&Ø!Ô(ð	
ñ 
ô 
ˆ	õ ”˜I°Ð@Ñ@Ô@ˆà˜AÒÐØ'¨(Ñ2°^ÑCÐØ•e”j Ð5EÐFÑFÔFÑFˆFà6<ˆ�s˜K¨Ð2Ñ3Ð3ð-=ð =r   ÚnumÚstatusÚnum_q_tokensÚmax_kv_readÚcachec           	      ór  ‡— ˆfd„t          | ¦  «        D ¦   «         }||z   }t          ||j        z  ¦  «        }g }|D ]y}	t          |	dg|z  d¬¦  «        }
‰|
_        dg|z  |
_        ||
_        |                     ||
j        d¦  «        }|€|c S | 	                    t          |
dd|¬¦  «        ¦  «         Œz|S )zPA utility function to create a list of FutureRequestStates for the warmup of CB.c                 ó,   •— g | ]}d ‰j         › d|› d�‘ŒS )Ú	__warmup_r   Ú__)Úname)Ú.0r[   rf   s     €r   ú
<listcomp>z/create_warmup_future_states.<locals>.<listcomp>°   s/   ø€ ÐGÐGÐG°qÐ2˜vœ{Ð2Ð2¨QÐ2Ð2Ð2ÐGÐGÐGr   r   r
   )Ú
request_idÚinitial_tokensÚmax_new_tokensNT)Úhas_new_tokenÚcomplete_blocksÚquery_length)rT   r   Ú
block_sizer   Ú_statusÚtokens_to_processÚposition_offsetÚallocate_blocksrq   Úappendr   )re   rf   rg   rh   ri   Úrequest_idsÚtotal_tokensÚblocks_neededÚfuture_statesÚreq_idÚstateÚ	allocateds    `          r   Úcreate_warmup_future_statesr„   §   sø   ø€ ð HÐGÐGÐG½EÀ#¹J¼JÐGÑGÔG€KØ +Ñ-€LÝ˜¨Ô(8Ñ8Ñ9Ô9€Mà€MØð 
ð 
ˆÝ¨À¸sÀ\Ñ?QÐbcÐdÑdÔdˆØˆŒØ#$ #¨Ñ"4ˆÔØ +ˆÔà×)Ò)¨-¸Ô9IÈ1ÑMÔMˆ	ØÐØ Ð Ð Ð Ø×ÒÝ˜u°DÈ!ÐZfÐgÑgÔgñ	
ô 	
ð 	
ð 	
ð Ðr   Úrequest_queuec                 óÞ   — g }|                       ¦   «         sV	 |                      ¦   «         }|�|                     |¦  «         n# t          j        $ r Y nw xY w|                       ¦   «         ¯V|S )z3Drains a queue and returns a list of RequestStates.)ÚemptyÚ
get_nowaitr|   ÚqueueÚEmpty)r…   Ú
new_statesr‚   s      r   Údrain_queuerŒ   Ä   sŒ   € à%'€JØ×!Ò!Ñ#Ô#ð ð	Ø!×,Ò,Ñ.Ô.ˆEØÐ Ø×!Ò! %Ñ(Ô(Ð(øøÝŒ{ð 	ð 	ð 	ØˆEð	øøøð ×!Ò!Ñ#Ô#ð ð Ðs   ˜+A ÁAÁAc                  óº   — t          d¦  «        r)t          j                             ¦   «         } | j        }| |fS d} t          j                             ¦   «         }| |fS )z®Returns a tuple of (mem_pool, graph_pool_id) for CUDA graphs. Since the MemPool object is only available in torch
    2.5+, we only return a graph_pool_id for older versions.z2.5.0N)r	   r+   r,   ÚMemPoolÚidÚgraph_pool_handle)Úmem_poolÚgraph_pool_ids     r   Úget_cuda_poolsr“   Ñ   s\   € õ ! Ñ)Ô)ð 'Ý”:×%Ò%Ñ'Ô'ˆØ œˆØ˜Ð&Ð&àˆÝœ
×4Ò4Ñ6Ô6ˆØ˜Ð&Ð&r   c              #   ó’   K  — | �>t           j                             | ¦  «        5  dV — ddd¦  «         dS # 1 swxY w Y   dS dV — dS )z’A context manager to use a CUDA mem pool. If the mem pool is None, it is a no-op. No type hint because it would
    make torch 2.4 or below crash.N)r+   r,   Úuse_mem_pool)r‘   s    r   Úmem_pool_ctxr–   Þ   s    è è € ð ÐÝŒZ×$Ò$ XÑ.Ô.ð 	ð 	ØˆEˆEˆEð	ð 	ð 	ñ 	ô 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	øøøð 	ð 	ð 	ð 	ð 	ð 	ð 	ˆˆˆˆˆs   ¤6¶:½:)r   )r
   )#r‰   Ú
contextlibr   Údataclassesr   Úmathr   r   Útypingr   r+   Ú transformers.configuration_utilsr   Útransformers.utilsr	   Úrequestsr   r   r   r   r0   Úboolr8   r*   r?   rE   rJ   ÚTensorÚlistrd   r„   ÚQueuerŒ   r“   r–   r.   r   r   ú<module>r¢      sÓ  ðð €€€Ø %Ð %Ð %Ð %Ð %Ð %Ø !Ð !Ð !Ð !Ð !Ð !Ø Ð Ð Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à €€€à =Ð =Ð =Ð =Ð =Ð =Ø 8Ð 8Ð 8Ð 8Ð 8Ð 8à EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ EÐ Eð#ð #ð #ð #ð #ñ #ô #ð #ð$ ðð ð ð ð ñ ô ñ „ððHÐ 0ð H°Tð Hð Hð Hð Hð
"˜#ð "¨cð "¸cð "Àcð "ð "ð "ð "ð"ð "�sð " sð "°sð "À3ð "ð "ð "ð "ð�cð  cð °Sð ¸Sð ð ð ð ð ð	Q=ð Q=Ø”LðQ=à˜sœ)ðQ=ð ˜sœ)ðQ=ð ð	Q=ð
 
ðQ=ð Q=ð Q=ð Q=ðhØ	ðàðð ðð ð	ð
 ðð 
Ð
Ôðð ð ð ð:
˜uœ{ð 
¨t°LÔ/Að 
ð 
ð 
ð 
ð
'ð 
'ð 
'ð ðð ñ „ðð ð r   