§
    ‚Štj%  ã                   óÒ   — d Z ddlZddlZddlZddlZddlmZ ddlZddlm	Z	 ddl
mZ ddlmZ  e	j        e¦  «        Z G d„ d	eej        ¦  «        Z G d
„ d¦  «        Zde_         dS )z+
CLI entry point for `transformers serve`.
é    N)Ú	Annotated)Úlogging)Úis_serve_availableé   )Úset_torch_seedc                   ó   — e Zd ZdZdZdZdS )ÚReasoningModeÚonÚoffÚautoN)Ú__name__Ú
__module__Ú__qualname__ÚONÚOFFÚAUTO© ó    úT/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/cli/serve.pyr	   r	   #   s   € € € € € Ø	€BØ
€CØ€D€D€Dr   r	   c            2       óþ  — e Zd Zdddddej        j        ddddddddddddddddfdeedz   ej	        d	¬
¦  «        f         dee
 ej        d¬
¦  «        f         deedz   ej        d¬
¦  «        f         dee
 ej        d¬
¦  «        f         deedz   ej        d¬
¦  «        f         dee ej        d¬
¦  «        f         deedz   ej        d¬
¦  «        f         dee ej        d¬
¦  «        f         deedz   ej        d¬
¦  «        f         dee
 ej        d¬
¦  «        f         dee ej        d¬
¦  «        f         deedz   ej        d ¬
¦  «        f         d!eedz   ej        d"¬
¦  «        f         d#eedz   ej        d$¬
¦  «        f         d%eedz   ej        d&¬
¦  «        f         d'ee
dz   ej        d(¬
¦  «        f         d)ee ej        d*¬
¦  «        f         d+ee ej        d,¬
¦  «        f         d-ee
 ej        d.¬
¦  «        f         d/ee ej        d0¬
¦  «        f         d1eedz   ej        d2¬
¦  «        f         d3ee
 ej        d4d5¬6¦  «        f         d7df.d8„Zd9„ Zd:„ Zd;„ ZdS )<ÚServeNFr   i,  Ú	localhosti@  ÚwarningÚforce_modelz*Model to preload and use for all requests.)ÚhelpÚcontinuous_batchingzMEnable continuous batching with paged attention. Configure with --cb-* flags.Úattn_implementationz2Attention implementation (e.g. flash_attention_2).Úcompilez*Enable torch.compile for faster inference.Úquantizationz.Quantization method: 'bnb-4bit' or 'bnb-8bit'.Ú	reasoninguÀ   Reasoning mode. 'auto' uses the chat template default. Only applies to models that support reasoning via their chat template (e.g. Qwen3, Gemma 4) â€” for other models this flag has no effect.Úchat_template_kwargsz‰Default JSON kwargs forwarded to apply_chat_template (e.g. '{"enable_thinking": true}'); per-request chat_template_kwargs override these.Údevicez4Device for inference (e.g. 'auto', 'cuda:0', 'cpu').Údtypez2Override model dtype. 'auto' derives from weights.Útrust_remote_codezTrust remote code when loading.Úmodel_timeoutzGSeconds before idle model is unloaded. Ignored when force_model is set.Úcb_block_sizez6KV cache block size in tokens for continuous batching.Úcb_num_blocksz2Number of KV cache blocks for continuous batching.Úcb_max_batch_tokensz1Maximum tokens per batch for continuous batching.Úcb_max_memory_percentz/Max GPU memory fraction for KV cache (0.0-1.0).Úcb_use_cuda_graphz+Enable CUDA graphs for continuous batching.ÚhostzServer listen address.ÚportzServer listen port.Úenable_corszEnable permissive CORS.Ú	log_levelz'Logging level (e.g. 'info', 'warning').Údefault_seedzDefault torch seed.Únon_blockingTz1Run server in a background thread. Used by tests.)Úhiddenr   Úreturnc           	      óø  — t          ¦   «         st          d¦  «        ‚dd l}ddlm} ddlm} ddlm} ddl	m
} ddlm} dd	lm} dd
lm} |�t#          |¦  «         t%          j        d¦  «        }|                     t$          j        |                     ¦   «                  ¦  «          |||	|
||||¬¦  «        | _        ddlm}  d„ |||||dœ                     ¦   «         D ¦   «         }!|!r | di |!¤Žnd }" ||||"¬¦  «        | _        |r>t9          j        |¦  «        }t=          |t>          ¦  «        stA          j!        d¦  «        ‚ni }|tD          j#        k    rd|d<   n|tD          j$        k    rd|d<    || j        | j        |¬¦  «        | _%         || j        | j        ¬¦  «        | _&         || j        | j        |¬¦  «        | _'         || j        | j        ¦  «        | _(         || j        | j%        | j&        | j'        | j(        | j        |¬¦  «        }#| )                    |#||d¬¦  «        }$| *                    |$¦  «        | _+        |r|  ,                    ¦   «          d S | j+         -                    ¦   «          d S )NzRMissing dependencies for serving. Install with `pip install transformers[serving]`r   r   )ÚChatCompletionHandler)ÚCompletionHandler)ÚModelManager)ÚResponseHandler)Úbuild_server)ÚTranscriptionHandler)ÚGenerationStateÚtransformers)r"   r#   r$   r   r   r%   r   )ÚContinuousBatchingConfigc                 ó   — i | ]
\  }}|®||“ŒS ©Nr   )Ú.0ÚkÚvs      r   ú
<dictcomp>z"Serve.__init__.<locals>.<dictcomp>Œ   s/   € ð 

ð 

ð 

á��1ð ˆ}ð ˆqð ˆ}ˆ}r   )Ú
block_sizeÚ
num_blocksÚmax_batch_tokensÚmax_memory_percentÚuse_cuda_graph)r   r   Ú	cb_configz,--chat-template-kwargs must be a JSON objectTÚenable_thinkingF)Úmodel_managerÚgeneration_stater!   )rJ   rK   )Úcompletion_handlerÚresponse_handlerÚtranscription_handlerrK   r-   Úinfo)r+   r,   r.   r   ).r   ÚImportErrorÚuvicornÚserving.chat_completionr4   Úserving.completionr5   Úserving.model_managerr6   Úserving.responser7   Úserving.serverr8   Úserving.transcriptionr9   Úserving.utilsr:   r   r   Ú
get_loggerÚsetLevelÚ
log_levelsÚlowerÚ_model_managerr;   r<   ÚitemsÚ_generation_stateÚjsonÚloadsÚ
isinstanceÚdictÚtyperÚBadParameterr	   r   r   Ú_chat_handlerÚ_completion_handlerÚ_response_handlerÚ_transcription_handlerÚConfigÚServerÚserverÚstart_serverÚrun)%Úselfr   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   rQ   r4   r5   r6   r7   r8   r9   r:   Útransformers_loggerr<   Ú	cb_kwargsrH   ÚappÚconfigs%                                        r   Ú__init__zServe.__init__*   s„  € õD "Ñ#Ô#ð 	tÝÐrÑsÔsÐsàˆˆˆàBÐBÐBÐBÐBÐBØ9Ð9Ð9Ð9Ð9Ð9Ø7Ð7Ð7Ð7Ð7Ð7Ø5Ð5Ð5Ð5Ð5Ð5Ø0Ð0Ð0Ð0Ð0Ð0Ø?Ð?Ð?Ð?Ð?Ð?Ø2Ð2Ð2Ð2Ð2Ð2ð Ð#Ý˜<Ñ(Ô(Ð(õ &Ô0°Ñ@Ô@ÐØ×$Ò$¥WÔ%7¸	¿ºÑ8IÔ8IÔ%JÑKÔKÐKà*˜lØØØ/Ø 3Ø%Ø'Ø#ð
ñ 
ô 
ˆÔð 	:Ð9Ð9Ð9Ð9Ð9ð

ð 

ð ,Ø+Ø$7Ø&;Ø"3ðð ÷ Še‰gŒgð

ñ 

ô 

ˆ	ð >GÐPÐ,Ð,Ð9Ð9¨yÐ9Ð9Ð9ÈDˆ	Ø!0 Ø 3ØØð"
ñ "
ô "
ˆÔð  ð 	&Ý#'¤:Ð.BÑ#CÔ#CÐ ÝÐ2µDÑ9Ô9ð YÝÔ(Ð)WÑXÔXÐXðYð $&Ð à�Ô(Ò(Ð(Ø6:Ð Ð!2Ñ3Ð3Ø�-Ô+Ò+Ð+Ø6;Ð Ð!2Ñ3à2Ð2ØÔ-Ø!Ô3Ø!5ð
ñ 
ô 
ˆÔð $5Ð#4ØÔ-Ø!Ô3ð$
ñ $
ô $
ˆÔ ð
 "1 ØÔ-Ø!Ô3Ø!5ð"
ñ "
ô "
ˆÔð ';Ð&:¸4Ô;NÐPTÔPfÑ&gÔ&gˆÔ#àˆlØÔØÔØ#Ô7Ø!Ô3Ø"&Ô"=Ø!Ô3Ø#ð
ñ 
ô 
ˆð —’ ¨$°TÀV�ÑLÔLˆØ—n’n VÑ,Ô,ˆŒàð 	Ø×ÒÑÔÐÐÐàŒK�OŠOÑÔÐÐÐr   c                 ó|   ‡ — ˆ fd„}t          j        |dd¬¦  «        ‰ _        ‰ j                             ¦   «          d S )Nc                  ó®   •— t          j        ¦   «         } t          j        | ¦  «         |                      ‰j                             ¦   «         ¦  «         d S r>   )ÚasyncioÚnew_event_loopÚset_event_loopÚrun_until_completerl   Úserve)Úloopro   s    €r   Ú_runz Serve.start_server.<locals>._runÐ   sJ   ø€ ÝÔ)Ñ+Ô+ˆDÝÔ" 4Ñ(Ô(Ð(Ø×#Ò# D¤K×$5Ò$5Ñ$7Ô$7Ñ8Ô8Ð8Ð8Ð8r   zuvicorn-threadF)ÚtargetÚnameÚdaemon)Ú	threadingÚThreadÚ_threadÚstart)ro   r}   s   ` r   rm   zServe.start_serverÏ   sS   ø€ ð	9ð 	9ð 	9ð 	9ð 	9õ
 !Ô'¨tÐ:JÐSXÐYÑYÔYˆŒØŒ×ÒÑÔÐÐÐr   c                 ó8   — | j                              ¦   «          dS )z$Clear all loaded models from memory.N)r]   Úshutdown©ro   s    r   Úreset_loaded_modelszServe.reset_loaded_modelsØ   s   € àÔ×$Ò$Ñ&Ô&Ð&Ð&Ð&r   c                 óü   — | j                              ¦   «          | j                             ¦   «          | j        r| j                             ¦   «         sd S d| j        _        | j                             d¬¦  «         d S )NTé   )Útimeout)r_   r†   r]   rƒ   Úis_aliverl   Úshould_exitÚjoinr‡   s    r   Úkill_serverzServe.kill_serverÜ   sy   € ØÔ×'Ò'Ñ)Ô)Ð)ØÔ×$Ò$Ñ&Ô&Ð&ØŒ|ð 	 4¤<×#8Ò#8Ñ#:Ô#:ð 	ØˆFØ"&ˆŒÔØŒ×Ò !ÐÑ$Ô$Ð$Ð$Ð$r   )r   r   r   r	   r   Úvaluer   Ústrrd   ÚArgumentÚboolÚOptionÚintÚfloatrt   rm   rˆ   r�   r   r   r   r   r   )   s‰  € € € € € ð quð
 ð Ødið ð ÔÔ$ð ØlrØpvØchð ð ð ð ð ð àLWØIMØUZØbkØX\ð ðAcð cà˜s T™z¨>¨5¬>Ð?kÐ+lÑ+lÔ+lÐlÔmðcð 'ØØˆEŒLÐmÐnÑnÔnðpô
ð	cð 'Ø�$‰J˜˜œÐ*^Ð_Ñ_Ô_Ð_ô
ðcð ˜4  ¤Ð3_Ð!`Ñ!`Ô!`Ð`Ôaðcð  Ø�$‰J˜˜œÐ*ZÐ[Ñ[Ô[Ð[ô
ðcð ØØˆEŒLð/ðñ ô ðô	
ðcð2 (Ø�$‰JØˆEŒLðmðñ ô ðô
ð3cðD ˜#˜|˜uœ|Ð1gÐhÑhÔhÐhÔiðEcðF ˜˜t™ \ U¤\Ð7kÐ%lÑ%lÔ%lÐlÔmðGcðH % T¨<¨5¬<Ð=^Ð+_Ñ+_Ô+_Ð%_Ô`ðIcðJ !Ø��”Ð#lÐmÑmÔmÐmô
ðKcðR !Ø�$‰J˜˜œÐ*bÐcÑcÔcÐcô
ðScðX !Ø�$‰J˜˜œÐ*^Ð_Ñ_Ô_Ð_ô
ðYcð^ 'Ø�$‰J˜˜œÐ*]Ð^Ñ^Ô^Ð^ô
ð_cðd  )Ø�D‰L˜,˜%œ,Ð,]Ð^Ñ^Ô^Ð^ô 
ðecðj %Ø�4‰K˜˜œÐ+XÐYÑYÔYÐYô
ðkcðr ˜˜\˜Uœ\Ð/GÐHÑHÔHÐHÔIðscðt ˜˜\˜Uœ\Ð/DÐEÑEÔEÐEÔFðucðv ˜t \ U¤\Ð7PÐ%QÑ%QÔ%QÐQÔRðwcðx ˜S , %¤,Ð4]Ð"^Ñ"^Ô"^Ð^Ô_ðycðz    d¡
¨L¨E¬LÐ>SÐ,TÑ,TÔ,TÐ TÔUð{cð|  Ø�,�%”, dÐ1dÐeÑeÔeÐeô
ð}cðB 
ðCcð cð cð cðJð ð ð'ð 'ð 'ð%ð %ð %ð %ð %r   r   uð  
Run a FastAPI server to serve models on-demand with an OpenAI compatible API.
Models will be loaded and unloaded automatically based on usage and a timeout.


Endpoints:
    POST /v1/chat/completions â€” Chat completions (streaming + non-streaming).
    POST /v1/completions      â€” Legacy text completions from a prompt.
    GET  /v1/models           â€” Lists available models.
    GET  /health              â€” Health check.

Requires FastAPI and Uvicorn: pip install transformers[serving]
)Ú__doc__rw   Úenumr`   r�   Útypingr   rd   Útransformers.utilsr   Útransformers.utils.import_utilsr   rX   r   rY   r   Úloggerr‘   ÚEnumr	   r   r   r   r   ú<module>rž      s	  ððð ð €€€Ø €€€Ø €€€Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð à €€€à &Ð &Ð &Ð &Ð &Ð &Ø >Ð >Ð >Ð >Ð >Ð >à )Ð )Ð )Ð )Ð )Ð )ð 
ˆÔ	˜HÑ	%Ô	%€ðð ð ð ð �C˜œñ ô ð ðy%ð y%ð y%ð y%ð y%ñ y%ô y%ð y%ðx€„€€r   