tjhddlZddlmZmZddlmZddlZddlmZddl mcm Z ddl m ZddlmZddlmZddlmZdd lmZmZdd lmZdd lmZmZmZmZdd lm Z dd l!m"Z"ddl#m$Z$ej%e&Z'eeGddeZ(eeGddeZ)Gddej*Z+Gddej*Z,edGddej*Z-Gddej*Z. d2dej*dej/d ej/d!ej/d"ej/dzd#e0d$e0e1zd%eefd&Z2Gd'd(ej*Z3Gd)d*ej*Z4eGd+d,eZ5eGd-d.e5Z6Gd/d0e5Z7gd1Z8dS)3N)CallableSequence) dataclass)initialization)use_kernel_forward_from_hub)FlashAttentionKwargs)BaseModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)capture_outputs) TimesFmConfigcPeZdZUdZdZejdzed<dZejdzed<dS) TimesFmOutputz loc (`torch.Tensor` of shape `(batch_size, )`): The mean of the time series inputs. scale (`torch.Tensor` of shape `(batch_size,)`): The scale of the time series inputs. Nlocscale) __name__ __module__ __qualname____doc__rtorchTensor__annotations__rj/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/timesfm/modeling_timesfm.pyrr,sL $C ###!%E5<$ %%%%%r"rcteZdZUdZdZejdzed<dZejdzed<dZ eje zdzed<dS)TimesFmOutputForPredictiona mean_predictions (`torch.Tensor` of shape `(batch_size, sequence_length)`): The mean predictions of the time series. full_predictions (`torch.Tensor` of shape `(batch_size, sequence_length)`): The full predictions of the time series including the mean and the quantiles. loss (`torch.Tensor` of shape `(1,)`, *optional*, returned when `future_values` is provided): The loss of the TimesFM model. Nmean_predictionsfull_predictionsloss) rrrrr&rrr r'r(floatr!r"r#r%r%:sj-1elT)000,0elT)000(,D%,  %,,,,,r"r%c0eZdZdZdeffd ZddZxZS) TimesFmMLPzPax MLP in pytorch.configct|j}|j}t j|||_t j|||_t j|d|_ dS)Nư>)normalized_shapeeps) super__init__ hidden_sizeintermediate_sizennLinear gate_proj down_proj LayerNorm layer_norm)selfr,r3r4 __class__s r#r2zTimesFmMLP.__init__Nsl ( "4;0ABB#4kBB, NNNr"Nc||}||}tj|}||}||d|dddddfz z}||zS)N?)r:r7Frelur8)r;xpaddingsgate_inpgateoutputss r#forwardzTimesFmMLP.forwardWst??1%%~~h''vd||..&&  x111d ';!;t||_||_||_t j|||_t j|_ t j|||_ t j|||_ dSrG) r1r2 input_dims hidden_dims output_dimsr5r6 input_layerSiLU activation output_layerresidual_layer)r;rNrOrPr<s r#r2zTimesFmResidualBlock.__init__ds $&&9Z=='))Ik;?? i K@@r"c||}||}||}||}||zSrG)rQrSrTrU)r;rAhiddenoutputresiduals r#rFzTimesFmResidualBlock.forwardosW!!!$$((""6**&&q))  r")rrrrr2rFrIrJs@r#rLrLasR!! A A A A A!!!!!!!r"rLRMSNormcTeZdZd deddffd ZdejdejfdZdZxZ S) TimesFmRMSNormr.r0returnNcttjt j||_||_dS)z= TimesFmRMSNorm is equivalent to T5LayerNorm N)r1r2r5 Parameterronesweightvariance_epsilon)r;r3r0r<s r#r2zTimesFmRMSNorm.__init__ysD l5:k#:#:;; #r" hidden_statesc|j}|tj}|ddd}|tj||jzz}|j||zS)NT)keepdim) dtypetorfloat32powmeanrsqrtrbra)r;rc input_dtypevariances r#rFzTimesFmRMSNorm.forwards|#) %((77  $$Q'',,R,>>% Ht?T4T(U(UU {]--k::::r"cHt|jjd|jS)Nz, eps=)tuplerashaperb)r;s r# extra_reprzTimesFmRMSNorm.extra_reprs& )**II$2GIIIr")r.) rrrr)r2rrrFrsrIrJs@r#r\r\ws$$$$$$$$$$;U\;el;;;;JJJJJJJr"r\c0eZdZdZdeffd ZddZxZS)TimesFmPositionalEmbeddingz6Generates position embedding for a given 1-d sequence.r,c t|j}|j}||c|_|_|j|_|jdz}t jt|t|z t|dz dz }| d|tj tj |tj| zzdS)Nrerinv_timescalesrh)r1r2 min_timescale max_timescaler3embedding_dimsmathlogr)maxregister_bufferrexparangerj)r;r,ryrznum_timescaleslog_timescale_incrementr<s r#r2z#TimesFmPositionalEmbedding.__init__s , , 1> .D.$0,1"&(5+?+?% BVBV+V"W"WZ]^lop^prsZtZt"t   EIel>&W&W&W[rZr&rss s     r"Nc||td|?tj|tj|jjd}n"|jdkrtd|j|j g|jdR|j dddz}tj tj |tj |gd }tj|ddd|jdzf}|S) aGenerates a Tensor of sinusoids with different frequencies. Args: seq_length: an optional Python int defining the output sequence length. if the `position` argument is specified. position: [B, seq_length], optional position for each token in the sequence, only required when the sequence is packed. Returns: [B, seqlen, D] if `position` is specified, else [1, seqlen, D] Nz.Either position or seq_length must be providedrhdevicerrez*position must be 2-dimensional, got shape rrfdim) ValueErrorrrrjrwr unsqueezendimrrviewcatsincosr?padr{)r; seq_lengthposition scaled_timesignals r#rFz"TimesFmPositionalEmbedding.forwards    2MNN N  |JemDL_LfgggqqrsttHH ]a  Z(.ZZ[[ [#hm7X^7Q777$:M:R:RSTVWY[:\:\\ EIk22EIk4J4JKQRSSSv1a)@PQQ r"queryr]ctj|jdt j|jz }||dddddfzS)Ng^$3eG?)r?softplusrmulr|sqrtr)r;rrs r# _scale_queryzTimesFmAttention._scale_querysO 4<((,,[49T];S;S-STTuT4qqq0111r"Nrcrrc |jdd}g|d|jR}|||dd}||}|||dd}|||dd}tj |j j t} | |||||f|j sdn|jdd|\} } | jg|dR} || } | | fS)Nrfrrerr>)rr)rrrrrrrrrr get_interfacer,_attn_implementationrrrreshaperr) r;rcrr input_shape hidden_shaperrrattention_interfacerrs r#rFzTimesFmAttention.forwards $)#2#. 88b8$-88 {{=1166|DDNNqRSTT ((66 [[//44\BBLLQPQRR {{=1166|DDNNqRSTT (?(M K ,.L) ) %8$7      % $}HCC$2H % %  % % ! \*k);;;;;;FFHH kk+.. L((r"rG)rrrrrintr2rrrr r rqrFrIrJs@r#rrsvvR}RRRRRRR(2%,25<2222/3))|) t+)-. ) u|U\D00 1 ))))))))r"rcleZdZdZdedeffd Zdejdejdejdejfd Z xZ S) TimesFmDecoderLayerzTransformer layer.r,rctt|||_t ||_t |j|j|_ dS)N)r)r0) r1r2r self_attnr+mlpr\r3 rms_norm_epsinput_layernormrs r#r2zTimesFmDecoderLayer.__init__s] )&IFFFf%%-f.@fFYZZZr"rcrrBr]c |}||}|||\}}||z}|||}|S)N)rcr)rB)rrr)r;rcrrBrrY_s r#rFzTimesFmDecoderLayer.forwardsh!,,];; >>')*   q!=0 BB r") rrrrrrr2rrrFrIrJs@r#rr s[}[[[[[[[| ,   r"rcteZdZUeed<dZdgZdZdZdZ e e dZ e jfdZxZS) TimesFmPreTrainedModelr,timesfmr past_values)timeT)rc attentionsc 4t|t|trt j|jdSt|tr|jdz}|j |j }}tj t|t|z t|dz dz }t j|j|t#jt#j|t"j| zzdSdS)Nrerrx)r1 _init_weights isinstancerinitones_rrur{rzryr|r}r)r~copy_rwrrrrj)r;rrrzryrr<s r#rz$TimesFmPreTrainedModel._init_weights9s f%%% f. / /  Jv~ & & & & &  : ; ; #2a7N+1+?AU=M&*hu]/C/CeMFZFZ/Z&[&[^a"A__' # J%)ELu}MMMQhPhhiij       r")rrrrr base_model_prefix_no_split_modulesmain_input_nameinput_modalities_supports_sdparr_can_record_outputsrno_gradrrIrJs@r#rr,s !./#O N,& U]___r"rceZdZdeffd ZdejdejdeejeejejfffdZe e e dejdej d ejd e edef d Ze dd ejdzdedejdejdedejdzf dZedejdejdeejejffdZedejdejdejfdZxZS) TimesFmModelr,ct|_tdjzjj|_tj j j|_ tj fdtjD|_|jjrt#|_|dS)NrerNrPrO)num_embeddings embedding_dimc0g|]}t|Sr!)r).0rr,s r# z)TimesFmModel.__init__..Ys$ e e e  3 3 e e er")r,)r1r2r,rL patch_lengthr3r4input_ff_layerr5 Embedding freq_sizefreq_emb ModuleListrangenum_hidden_layerslayersuse_positional_embeddingru position_emb post_initr;r,r<s `r#r2zTimesFmModel.__init__Ns     26..*0     F4DTZTfggg m e e e eU6KcEdEd e e e    ; / J :& I I ID  r"inputs patched_padsr]c|||\}}tj||jj}||ddddfz |ddddfz }tjtj||jjz |jjktj|jj|j |j |}|||ffS)zInput is of shape [B, N, P].minNr) _timesfm_masked_mean_stdrclampr, tolerancewhereabspad_valtensorrhr)r;rrmusigmarEs r#_forward_transformzTimesFmModel._forward_transformas11&,GG E Et{'<===Bqqq$}--qqq$}1EE+ Ift{22 3 3dk6K K L,GM'. Y Y Y    U ##r"rpast_values_paddingfreqrc |jd}||d|jj}||d|jj}t jt j|dz |jjkt jd|j |j |}t jt j||jj z |jjkt jd|j |j |}| ||\}}|d|z z}t j ||gd} || } t j|dd} |jjr`|| jd} t j| g| jdzd} || | } | | z } ||} | | z } | }|| |jd|j |j d }|jd |jjD]}||f|| d |}t1||d|d S) a past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Past values of the time series that serves as input to the model. past_values_padding (`torch.LongTensor` of shape `(batch_size, sequence_length)`): The padding indicator of the time series. freq (`torch.LongTensor` of shape `(batch_size,)`): Frequency indices for the time series data. rrfr>rrrrT)rsequence_lengthrhrrN)rrB)last_hidden_staterr)rrrr,rrrrrrrhrrr rrrrrconcat_timesfm_shift_padded_seqr_prepare_4d_attention_maskrrr)r;rr r rbsizepatched_inputsrstats concat_inputs model_inputpatched_paddingpos_embf_embrcrlayers r#rFzTimesFmModel.forwardqs&!!$$))%T[5MNN*//r4;;STT  IlS( ) )DK,A A LN$8AV W W W    { Int{':: ; ;dk>S S LL$6|?R S S S   !% 7 7 U U(3+=> ><"@bIII ))-88  )Lb999!< ; / #'' (9!(<==GlG9{/@/C#CKKKG44_gNNG 7 "K d##u $ 88*)/2% ' 9  [!@4;#@!@A  E!E-( MM+a(    r"TrNr rhrrc|jrtj|jntj|j}|(||jdddd}||z}|rbtjtj||f|||zd}|dd||}|tj ||}n|}|S)a Creates 4D attention mask and combines causal and padding masks if needed. Args: attention_mask: Optional tensor of shape (batch_size, seq_length) containing padding mask sequence_length: Length of the sequence dtype: Data type of the mask device: Device of the mask is_causal: Whether to apply causal masking Returns: 4D attention mask of shape (batch_size, 1, seq_length, seq_length) Nrrrfr)diagonal) is_floating_pointrfinforiinforrrtriur`minimum)rr rhrr min_value causal_masks r#rz'TimesFmModel._prepare_4d_attention_masks,/4.EaEK&&**5;W\K]K]Ka   %+001Ea1H!QPRSSN+i7N  -* O_=USYZZZ]ffK&**1a/RRK)!&~{!K!K!,r"paddingcDdtjfd}tjd|z d}||}tj|jd}|||ddf}|||ddf}d|z }tj|d} tj| d } tj||zd} | | z } || d z |z} tj| dzd| z } tj| d } tj| }| |fS) aCalculates mean and standard deviation of `inputs` across axis 1. It excludes values where `padding` is 1. Args: inputs: A PyTorch tensor of shape [b, n, p]. padding: A PyTorch tensor of shape [b, n, p] with values 0 or 1. Returns: A tuple containing the mean and standard deviation. We return the statistics of the first patch with more than three non-padded values. arrc.tj|dktjd}|dktjd}tj|dk|jddz |S)Nrrrr)rargmaxriint32sumrrr)r&indicesrow_sums r#_get_patch_indexz?TimesFmModel._timesfm_masked_mean_std.._get_patch_indexsslC1H==#=#=1EEEGaxmmEK00444;;G;w!|SYq\A-=wGG Gr"rrerrNr>rrfr)rrr*rrrrrr)rr$r-pad_sum patch_indicesbidxsr&rmasknum_valid_elements masked_sum masked_meanmasked_centered_arr masked_var masked_stds r#rz%TimesFmModel._timesfm_masked_mean_stdsX  H%, H H H H )AKQ///((11  V\!_--UM111,-e]AAA-.3w#Yt333"[);EEEYsTzq111  #55  #[%:%:2%>%>>$FY2A51===@RR [555 Z ++ J&&r"r1seqc|j\}}}|dk}|tjd}d||d<tj||jddd |d|}||ddddfz |z}| d|} | S)zShifts rows of seq based on the first 0 in each row of the mask. Args: mask: mask tensor of shape [B, N] seq: seq tensor of shape [B, N, P] Returns: The shifted sequence. rrrrf)rN) rrrirr)r(anyrrrexpandgather) r1r8 batch_sizenum_seq feature_dimnew_maskr+ idx_range shifted_idx shifted_seqs r#rz&TimesFmModel._timesfm_shift_padded_seqs,/9( G[%)QY++ek**11a188)+!$$$%L<<<AA!RKKRRS]_acnoo !7111dD=#99WD jjK00 r")T)rrrrr2rrrqr rrr LongTensorr rrrF staticmethodrrhrboolrrrrIrJs@r#rrLs}&$l$27,$ u|U5<#=>> ?$$$$  F \F #-F l F +, F  F F F ^_ F P  ++ t+++{+ +  +   +++\+Z,',' ,'QVW\WcejeqWqQr,',','\,'\ 5<EL\r"rceZdZdZdeffd Z ddeejdee dzde dzde ejd ffd Z d ejd e ejejfdejfd Z dejdejdejfdZ ee ddeejdeeje zdzde dzdejdzde dzdededeedefdZedejde deejfdZxZS)TimesFmModelForPredictionz/TimesFM model for quantile and mean prediction.r,cTt|||_|j|_|j|_t||_t|j |jdt|j zz|j |_|dS)Nrr)r1r2r,context_length context_lenhorizon_length horizon_lenrdecoderrLr3len quantilesr4horizon_ff_layerrrs r#r2z"TimesFmModelForPrediction.__init__=s     !0!0#F++ !5)-S9I5J5J1JK0! ! !  r"Nrr rKr].c ||j}gg}}|D]}|jd}tj||jz|j|j}||krt||z } tjtj| |j|j|gd}tjtj| |j|j|gd}n$||kr|| d}|||jz d}| || |tj |dtj |df} |M| tj |dt|tj ddfz} | S)aPad/truncate input time series to `context_len` and build a padding mask. Args: inputs: A list of 1d Tensors. Each Tensor is the context time series of a single forecast task. freq: Optional list of frequencies (returned as a tensor when provided). context_len: Optional context length override (defaults to `self.context_len`). Returns: Tuple of (padded_inputs, padding_mask) and optionally a freq tensor. Nrrrrxrfr)rKrrrzerosrMrhrrr`appendstackrrOr)r) r;rr rKinput_ts input_paddingts input_lenr$ num_front_padresults r# _preprocessz%TimesFmModelForPrediction._preprocessPs  *K"$b- * *B Ik)d.>">bhWYW`aaaG;&& +i 7 Y MRTR[ \ \ \^`aghiii)UZ RXV]Vd%e%e%egn$ouvwww[(( &!K$2B$B"C"E"EF OOB      ) ) ) )+hA... Mq0Q0Q0QR  u|D3v;;,?u{SSS[[\^`abbddF r" model_outputrc||}|j\}}}||||jjt |jjdz}|\}}||dddddfz|dddddfzS)z*Postprocess output of stacked transformer.rN)rQrrrr,rLrOrP) r;r]r output_tsbnrrrs r#_postprocess_outputz-TimesFmModelForPrediction._postprocess_outputus )),77 /1aNN1a)CSI^E_E_bcEcdd  E5D$!455111dD$;N8OOOr" predictionstargetsc4g}t|jjD]W\}}||d|fz }tj|dz |z||z}||Xtj|S)N.r) enumerater,rPrr~rTrlrU)r;rcrdlossesiqerrorsr(s r#_quantile_lossz(TimesFmModelForPrediction._quantile_losssdk344 ' 'DAq{3622F9a!ev-q6z::D MM$))++ & & & &{6""'')))r"Fr window_size future_valuesforecast_context_lenreturn_forecast_on_contexttruncate_negativerc  #||j#n|#|dj} #fd|D} tjtjd| D} |ig} g} t | D]O\}}| ||||| ||gdzP| } || }|-t ddgt| z}| | |\}}}| | }| | }| | }|}|j d}g}|j d|j d|jzkr3td|j dd |j dd |j|jj}|j|zdz |z}t%|D]9}|ddd|j df}|dd# df}|dd# df}|jd|||d |}||j|j|jf}|rv|dkrp|dddd d|jjddf}||dd |d }|||ddd d|df}|ddd d|ddf}||tj||gd };|r;tj|dddd||jjz |jzddf}n*tj|dddd|jddf}|dddddf}|6|ddddf|ddddfz}|ddddf|ddddfz}|rX| dk}tj||d|}tj||d|}d} |?t?j ||}!|!|ddddddf|}"|!|"z} tE|j|j#|j$||| S)a past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Past values of the time series that serves as input to the model. freq (`torch.LongTensor` of shape `(batch_size,)`): Frequency indices for the time series data. window_size (`int`, *optional*): Window size of trend + residual decomposition. If None then we do not do decomposition. future_values (`torch.Tensor`, *optional*): Optional future time series values to be used for loss computation. forecast_context_len (`int`, *optional*): Optional max context length. return_forecast_on_context (`bool`, *optional*): True to return the forecast on the context when available, i.e. after the first input patch. truncate_negative (`bool`, *optional*): Truncate to only non-negative values if any of the contexts have non-negative values, otherwise do nothing. Example: ```python >>> from transformers import TimesFmModelForPrediction >>> model = TimesFmModelForPrediction.from_pretrained("google/timesfm-2.0-500m-pytorch") >>> forecast_input = [torch.linspace(0, 20, 100).sin(), torch.linspace(0, 20, 200).sin(), torch.linspace(0, 20, 400).sin()] >>> frequency_input = torch.tensor([0, 1, 2], dtype=torch.long) >>> # Generate >>> with torch.no_grad(): >>> outputs = model(past_values=forecast_input, freq=frequency_input, return_dict=True) >>> point_forecast_conv = outputs.mean_predictions >>> quantile_forecast_conv = outputs.full_predictions ``` Nrc&g|] }| dSrGr!)rrX fcontext_lens r#rz5TimesFmModelForPrediction.forward..s$;;;"l]^^$;;;r"c6g|]}tj|Sr!)rr)rrXs r#rz5TimesFmModelForPrediction.forward..s (H(H(H22(H(H(Hr"rez6No frequency provided via `freq`. Default to high (0).rz=Length of paddings must match length of input + horizon_len: z != z + )rr r rfr)axis.r)rrrcr&r'r(r!)%rKrrrrUrfextend_timesfm_moving_averageloggerinforOr\rirrrMrr,rLrrNrbrrrrrsizerT concatenater clamp_minr?mse_lossrkr%rrc)$r;rr rlrmrnrorprrrinp_min new_inputs new_freqsrhrXrVrWinp_freq final_outrK full_outputsoutput_patch_lennum_decode_patches step_indexcurrent_paddingdecoder_output fprop_outputs new_full_tsnew_ts mean_outputsrr(r} quantile_lossrss$ @r#rFz!TimesFmModelForPrediction.forwards^ '+LL/LQ&;;;;{;;;)EK(H(H(H(H(HIIJJ  "JI"6** 4 42!!$">">r;"O"OPPP#$$d1gY]333F  < KKP Q Q Q3V$D,0,<,M,8DL-$$1-- --N !440#^%9:M * 1jAoo+AAAss4Ndk6N4NPQPQPQ,QR )11+2B2B12E2Er;K[K[\]K^K^__ ##K000"111b*;+;*;Q#>?F'2/@0@/@!!!(CDK    , , ,)9f*=BGGGII % _ ,\BBBPkDK$<>H // QQQ122X0F VVMm+D),>%0(6))     r"r&c2tj||dz dfdd}tj||j|j|z }tj|ddd|ddd}|||z gS)zCCalculates the moving average using PyTorch's convolution function.rrconstantrrf) r?rrr`rhrconv1drsqueeze)r&rl arr_paddedkernel smoothed_arrs r#rwz1TimesFmModelForPrediction._timesfm_moving_averagesU3q! 4j!DD KsyLLL{Zx 1b 9 96;;q!R;P;PQQYY[[ cL011r"r)NNNNFF)rrrrrr2rrrrrqr\rbrkrrrFr rr%rFrElistrwrIrJs@r#rHrH:s999}(lp##u|,#4rs:* ........!!!!!! &&&&&&777777BBBBBB//////FFFFFFFF&&&&&&RRRRRRRRRRRR777777555555000000  H % %  & & & & &O & &  &  - - - - - - -  -,!!!!!29!!!,Y''JJJJJRYJJ('J(++++++++j%% I%,% %, % L4' %  %S[%'(%%%%,9)9)9)9)9)ry9)9)9)x")@_>jjjjj)jjjZm2m2m2m2m2 6m2m2m2` R Q Qr"