hh!ddlZddlmZmZddlmZddlmZddlZddl m Z ddl m Z e jdZed eej"eeeffd ZGd d eZeGd dZeGddZy)N) dataclassfield)Enum)Optional)logging)tracedContinuousBatchingLoggerreturnc@tjjrtjd}tjj tjj tjj |j}tjj|}tjj|}ntjjjrtjjjrWtjd}tjj}|tjjz }d}ntjd}d}d}d}||||fS)Ncudampsrcpu)torchr is_availabledevice empty_cache synchronizeget_device_properties total_memorymemory_reservedmemory_allocatedbackendsris_builtdriver_allocated_memoryrecommended_max_memory)rrreserved_memoryallocated_memorys s/var/www/html/aiagenthome/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/requests.pyget_device_and_memory_breakdownr s zz f%    zz77?LL **44V< ::66v>    ( ( *u~~/A/A/J/J/Le$yy88: '%))*J*J*LLe$  <2B BBc,eZdZdZdZdZdZdZdZdZ dZ y ) RequestStatusz5Status of a generation request through its lifecycle.pending prefillingprefilling_splitsplit_pending_remainderdecodingfinishedfailedN) __name__ __module__ __qualname____doc__PENDING PREFILLINGPREFILLING_SPLITSPLIT_PENDING_REMAINDERDECODINGFINISHEDFAILEDr!rr#r#6s*?GJ)7HH Fr!r#ceZdZUdZeed<eeZee ed<eeZ ee ed<eeZ ee ed<dZ eeed<ej Zeed <eej$Ze ed <y) GenerationOutputa5Tracks the output of a generation request. Attributes: request_id (str): The ID of the generation request. prompt_ids (list[int]): The IDs of the prompt tokens. generated_tokens (list[int]): The generated tokens. logprobs (list[float]): The log probabilities of the generated tokens. error (Optional[str]): Any error message associated with the request. When None, the request was successful. status (RequestStatus): The status of the request. created_time (float): The time the request was created. request_iddefault_factory prompt_idsgenerated_tokenslogprobsNerrorstatus created_time)r+r,r-r.str__annotations__rlistr<intr=r>floatr?rr#r/r@timerAr6r!rr8r8Bsy O!$7JS 7"'"=d3i=!$7Hd5k7E8C=)11FM1 :L%:r!r8ceZdZUdZeed<dZeee ed<dZ eee ed<e eZ ee ed<e eZ ee ed<d Ze ed <d Ze ed <ej"Zeed <d Ze ed<dZe ed<e ej*Zeed<dZeeed<dZeeefed<edefdZej:defdZdZde fdZde fdZ e!de de"fdZ#dZ$dZ%y) RequestStateayTracks the state of a generation request through its lifecycle. Attributes: request_id (str): The ID of the generation request. full_prompt_ids (list[int] | None): The tokens IDs of the full prompt. prompt_ids (list[int] | None): The tokens IDs currently being processed. remaining_prompt_ids (list[int]): The tokens IDs remaining to be processed (for split requests). static_outputs (list[int]): The generated tokens. allocated_blocks (int): The number of blocks allocated to the request. position_offset (int): The current position in the sequence for position_ids. status (RequestStatus): The status of the request: can be one of PENDING, PREFILLING, PREFILLING_SPLIT, SPLIT_PENDING_REMAINDER, DECODING, FINISHED, FAILED max_new_tokens (int): The maximum number of new tokens to generate. eos_token_id (int): The ID of the end-of-sequence token. created_time (float): The time the request was created. error (Optional[str]): Any error message associated with the request. When None, has had no error yet. r9Nfull_prompt_idsr<r:remaining_prompt_idsstatic_outputsrallocated_blocksposition_offset_statusmax_new_tokens eos_token_idrAr?)rRrRlifespanr c|jS)N)rOselfs rr@zRequestState.status|s ||r!valuec(|jtjk(r#tjdf|_||_y|tj k(r8|jdtjf|_|j ||_y)NrRr)rOr#r/rGrTr4log_end_of_request)rWrXs rr@zRequestState.statussm <<=00 0!YY["-DM m,, ,!]]1-tyy{;DM  # # % r!c t|j}|j}|jd|jz }|jd|jz }t j d|jd|d|d|d| y)NrzRequest z finished: prefill_len = z decode_len = z start_time = z end_time = )lenrJ generated_lenrTrAloggerinfor9)rW prefill_len decode_len start_timeend_times rrZzRequestState.log_end_of_requests$../ '') ]]1%(9(99 ==#d&7&77 t''A;2B/J?RaT^Sbbodlcp q r!c|jS)zCGet the current length of the sequence (prompt + generated tokens).)rNrVs r current_lenzRequestState.current_lens###r!c,t|jS)z*Get the number of tokens generated so far.)r]rLrVs rr^zRequestState.generated_lens4&&''r!token_idc.|jtjk7ry||jk(xr|jdk7}|j |j k\}|r|r|j j|g|s|rtj|_yy)zUpdate the request with a newly generated token and check for completion. Args: token_id: The token ID to add to the output sequence Returns: bool: True if the request is now complete, False otherwise FrRT) r@r#r3rSr^rQrLextendr4)rWrhis_eos is_max_lens rupdate_with_tokenzRequestState.update_with_tokens ;;-00 0T...J43D3D3J'')T-@-@@ 6    & &z 2 Z'00DKr!c nd|jd|jd|jdt|jdt|j d|j dt|jd|jd |jg }d d j|zd zS) Nz request_id=zstatus=z out_tokens=z query_length=zremaining_tokens=z kv_length=zfull_prompt_length=zallocated_blocks=zgenerated_tokens=zRequestState( z, z )) r9rOr^r]r<rKrNrJrMrLjoin)rWmsgs r__repr__zRequestState.__repr__s$//* +dll^ $$,,./ 0C01 2D$=$= >? @--. /!#d&:&:";!< = 5 56 7 3 34 5  #W\\#%66>>r!ct|j|j|j|jg|j S)z7Convert the request state to a GenerationOutput object.)r9r<r@r=r>r?)r8r9rJr@rLr?rVs rto_generation_outputz!RequestState.to_generation_outputs9++;;!00**   r!)&r+r,r-r.rBrCrJrrDrEr<rrKrLrMrNr#r/rOrQrSrGrArFr?rTtuplepropertyr@setterrZrfr^r boolrmrqrsr6r!rrIrIYsI&O+/OXd3i(/&*Jc#*&+D&A$s)A %d ;NDI;cOS*22G]2NCL# :L%:E8C=$,HeE5L!,   ]]M $S$(s(  #$ 4 ?  r!rI)rG dataclassesrrenumrtypingrr utils.loggingr utils.metricsr getLoggerr_ staticmethodrtrrEr r#r8rIr6r!rrs ( $#   5 6Cu||S#s/J)KCC, D  ;; ;, s s  s r!