§
    ‚Štjš  ã                   ó˜   — d dl Z d dlZd dlZddlmZ ddlmZ ddlmZ  ej	        e
¦  «        Z G d„ d¦  «        Z G d„ d	e¦  «        ZdS )
é    Né   )ÚTrainerCallback)ÚPREFIX_CHECKPOINT_DIR)Úloggingc                   ó4   — e Zd Zd	defd„Zd„ Zd„ Zd„ Zd„ ZdS )
ÚCheckpointManageré   Ú	kill_waitc                 ó>   — || _         d| _        d| _        || _        dS )aD  
        Initialize the CheckpointManager for Just-In-Time checkpoint handling.

        Args:
            trainer: The Trainer instance that will be used to save checkpoints when SIGTERM is received.
            kill_wait (`int`, *optional*, defaults to 3): Grace period to distinguish between SIGTERM and SIGKILL.
        FN)ÚtrainerÚis_checkpoint_requestedÚ_original_sigterm_handlerr
   )Úselfr   r
   s      úa/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/trainer_jit_checkpoint.pyÚ__init__zCheckpointManager.__init__   s&   € ð ˆŒØ',ˆÔ$Ø)-ˆÔ&Ø"ˆŒˆˆó    c                 óŒ   — t          j         t           j        | j        ¦  «        | _        t                               d¦  «         d S )Nz4JIT checkpoint signal handler registered for SIGTERM)ÚsignalÚSIGTERMÚ_sigterm_handlerr   ÚloggerÚinfo©r   s    r   Úsetup_signal_handlerz&CheckpointManager.setup_signal_handler   s4   € Ý)/¬µv´~ÀtÔG\Ñ)]Ô)]ˆÔ&Ý�ŠÐJÑKÔKÐKÐKÐKr   c                 óÀ   — | j         rd S t                               d| j        › d�¦  «         t	          j        | j        | j        ¦  «                             ¦   «          d S )Nz4SIGTERM received, will request JIT checkpoint after Ús)r   r   r   r
   Ú	threadingÚTimerÚ_enable_checkpointÚstart)r   ÚsignumÚframes      r   r   z"CheckpointManager._sigterm_handler   s[   € ØÔ'ð 	ØˆFå�ŠÐ\È4Ì>Ð\Ð\Ð\Ñ]Ô]Ð]ÝŒ˜œ¨Ô(?Ñ@Ô@×FÒFÑHÔHÐHÐHÐHr   c                 óH   — t                                d¦  «         d| _        d S )Nz/Kill wait period elapsed, requesting checkpointT)r   r   r   r   s    r   r   z$CheckpointManager._enable_checkpoint&   s#   € Ý�ŠÐEÑFÔFÐFØ'+ˆÔ$Ð$Ð$r   c                 óØ  — 	 d| _         t                               d¦  «         | j        j        j        }t                               d|› �¦  «         | j                             d ¬¦  «        }t          › d|› �}t          j	         
                    ||¦  «        }t          j        |d¬¦  «         t          j	         
                    ||d¦  «        }t          |d	¦  «        5 }|                     d
|› d�¦  «         d d d ¦  «         n# 1 swxY w Y   t                               d|› �¦  «         | j                             | j        j        d ¬¦  «         t          j	                             |¦  «        r.t          j        |¦  «         t                               d¦  «         t                               d¦  «         d S # t$          $ r#}t                               d|› �¦  «         ‚ d }~ww xY w)NFzStarting JIT checkpointing...zSaving JIT checkpoint at step )Útrialú-T)Úexist_okzcheckpoint-is-incomplete.txtÚwzCheckpoint started at step z and in progress...z2Created checkpoint progress sentinel marker file: zSentinel marker file removedz/Immediate JIT checkpoint completed successfullyzFailed to save JIT checkpoint: )r   r   r   r   ÚstateÚglobal_stepÚ_get_output_dirr   ÚosÚpathÚjoinÚmakedirsÚopenÚwriteÚ_save_checkpointÚmodelÚexistsÚremoveÚ	ExceptionÚerror)r   Úcurrent_stepÚ
output_dirÚcheckpoint_folderÚcheckpoint_pathÚsentinel_fileÚfÚes           r   Úexecute_jit_checkpointz(CheckpointManager.execute_jit_checkpoint*   s  € ð!	à+0ˆDÔ(å�KŠKÐ7Ñ8Ô8Ð8Øœ<Ô-Ô9ˆLÝ�KŠKÐG¸ÐGÐGÑHÔHÐHàœ×5Ò5¸DÐ5ÑAÔAˆJÝ#8Ð IÐ I¸<Ð IÐ IÐÝ œgŸlšl¨:Ð7HÑIÔIˆOõ ŒK˜°$Ð7Ñ7Ô7Ð7õ œGŸLšL¨Ð5FÐHfÑgÔgˆMÝ�m SÑ)Ô)ð Y¨QØ—’ÐW°lÐWÐWÐWÑXÔXÐXðYð Yð Yñ Yô Yð Yð Yð Yð Yð Yð Yøøøð Yð Yð Yð Yå�KŠKÐ\È]Ð\Ð\Ñ]Ô]Ð]ð ŒL×)Ò)¨$¬,Ô*<ÀDÐ)ÑIÔIÐIõ Œw�~Š~˜mÑ,Ô,ð <Ý”	˜-Ñ(Ô(Ð(Ý—’Ð:Ñ;Ô;Ð;å�KŠKÐIÑJÔJÐJÐJÐJøåð 	ð 	ð 	Ý�LŠLÐ>¸1Ð>Ð>Ñ?Ô?Ð?Øøøøøð	øøøs=   ‚CF< ÃDÃ9F< ÄD	Ä	F< ÄD	ÄB-F< Æ<
G)ÇG$Ç$G)N)r	   )	Ú__name__Ú
__module__Ú__qualname__Úintr   r   r   r   r?   © r   r   r   r      sv   € € € € € ð#ð #¨3ð #ð #ð #ð #ðLð Lð LðIð Ið Ið,ð ,ð ,ð"ð "ð "ð "ð "r   r   c                   ó<   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
d	S )
ÚJITCheckpointCallbackaN  
    Callback for Just-In-Time checkpointing on SIGTERM signals.

    When SIGTERM is received, the checkpoint manager sets `is_checkpoint_requested=True`.
    The callbacks detect this flag and set `control.should_training_stop=True`, which signals
    the Trainer's training loop to exit gracefully after saving the checkpoint.
    c                 ó"   — d | _         d | _        d S )N)r   Újit_managerr   s    r   r   zJITCheckpointCallback.__init__X   s   € ØˆŒØ59ˆÔÐÐr   c                 óÀ   — || _         |j        j        rJt          |¬¦  «        | _        | j                             ¦   «          t                               d¦  «         d S d S )N)r   zJIT checkpointing enabled)r   ÚargsÚenable_jit_checkpointr   rH   r   r   r   )r   r   s     r   Úset_trainerz!JITCheckpointCallback.set_trainer\   sa   € ØˆŒØŒ<Ô-ð 	5Ý0¸ÐAÑAÔAˆDÔØÔ×1Ò1Ñ3Ô3Ð3Ý�KŠKÐ3Ñ4Ô4Ð4Ð4Ð4ð	5ð 	5r   c                 ót   — | j         r.| j         j        r$d|_        | j                              ¦   «          d S d S d S ©NT©rH   r   Úshould_training_stopr?   ©r   rJ   r)   ÚcontrolÚkwargss        r   Úon_pre_optimizer_stepz+JITCheckpointCallback.on_pre_optimizer_stepc   óQ   € ØÔð 	6 Ô 0Ô Hð 	6Ø+/ˆGÔ(ØÔ×3Ò3Ñ5Ô5Ð5Ð5Ð5ð	6ð 	6ð 	6ð 	6r   c                 ót   — | j         r.| j         j        r$d|_        | j                              ¦   «          d S d S d S rN   rO   rQ   s        r   Úon_step_beginz#JITCheckpointCallback.on_step_beginh   rU   r   c                 ó‚   — | j         r5| j         j        r+d|_        d|_        | j                              ¦   «          d S d S d S ©NFT©rH   r   Úshould_saverP   r?   rQ   s        r   Úon_step_endz!JITCheckpointCallback.on_step_endm   óY   € ØÔð 	6 Ô 0Ô Hð 	6Ø"'ˆGÔØ+/ˆGÔ(ØÔ×3Ò3Ñ5Ô5Ð5Ð5Ð5ð	6ð 	6ð 	6ð 	6r   c                 ó‚   — | j         r5| j         j        r+d|_        d|_        | j                              ¦   «          d S d S d S rY   rZ   rQ   s        r   Úon_epoch_endz"JITCheckpointCallback.on_epoch_ends   r]   r   c                 óº   — | j         rQ| j         j        �Gt          j        t          j        | j         j        ¦  «         t                               d¦  «         d S d S d S )Nz;Restored original SIGTERM handler after training completion)rH   r   r   r   r   r   rQ   s        r   Úon_train_endz"JITCheckpointCallback.on_train_endy   s^   € àÔð 	W Ô 0Ô JÐ VÝŒM�&œ.¨$Ô*:Ô*TÑUÔUÐUÝ�KŠKÐUÑVÔVÐVÐVÐVð	Wð 	WÐ VÐ Vr   N)r@   rA   rB   Ú__doc__r   rL   rT   rW   r\   r_   ra   rD   r   r   rF   rF   O   sŒ   € € € € € ðð ð:ð :ð :ð5ð 5ð 5ð6ð 6ð 6ð
6ð 6ð 6ð
6ð 6ð 6ð6ð 6ð 6ðWð Wð Wð Wð Wr   rF   )r,   r   r   Útrainer_callbackr   Útrainer_utilsr   Úutilsr   Ú
get_loggerr@   r   r   rF   rD   r   r   ú<module>rg      sÔ   ðØ 	€	€	€	Ø €€€Ø Ð Ð Ð à -Ð -Ð -Ð -Ð -Ð -Ø 0Ð 0Ð 0Ð 0Ð 0Ð 0Ø Ð Ð Ð Ð Ð ð 
ˆÔ	˜HÑ	%Ô	%€ð?ð ?ð ?ð ?ð ?ñ ?ô ?ð ?ðD.Wð .Wð .Wð .Wð .W˜Oñ .Wô .Wð .Wð .Wð .Wr   