[TU]5 X lXlX0l[ XR- 5Ul[R"X5Ul [R"X5Ul [R"X5Ul [R"X5Ul [5UlgN)super__init__ num_headsr layer_idxintdepthrLinearWqWkWvdenseset pruned_heads)selfrrTrU __class__s r rSMultiHeadAttention.__init__Ws{ "("67 ))L7))L7))L7YY|: Er"cURUR-n[U5S:Xag[XRX R5up[ UR U5Ul[ URU5Ul[ URU5Ul[ URUSS9Ul UR[U5- UlX R-UlURRU5Ulg)Nrrr') rrTlenrr^rrYrZr[r\union)r_headsattention_head_sizeindexs r prune_headsMultiHeadAttention.prune_headsfs"//4>>A u:? 7~~Obduduv %TWWe4$TWWe4$TWWe4' EqA #e*4/..@ --33E:r"cxURUSURUR5nUR/SQ5$)Nr&rrrr )reshaperTrWr9)r_x batch_sizes r split_into_heads#MultiHeadAttention.split_into_headsws- IIj"dnndjj Ayy&&r"c URSn URU5nURU5nURU5nUR X;5nUR X+5nUR X5nUb!UR X!UR SU 05up![X2XXg5n U SR/SQ5n U SnU RU SUR5nURU5n X4$)Nrcache_positionrkrr&) r:rYrZr[roupdaterUrLr9rlrr\)r_rAr@r?rB layer_pastrCrD use_cacheoutput_attentionsrrrnrKscaled_attentionattnoriginal_size_attentions r forwardMultiHeadAttention.forward{sWWQZ GGAJ GGAJ GGAJ  ! !! 0  ! !! 0  ! !! 0  !$$Q4>>[TU]5 [XUS9Ul[ X5Ul[ R"USS9Ul[ R"USS9Ul [ R"U5Ul [ R"U5Ul g)NrUgư>eps) rRrSrNmulti_head_attentionrffnr LayerNorm layernorm1 layernorm2Dropoutdropout1dropout2)r_rrTrraterUr`s r rSEncoderLayer.__init__sn $6|Zc$d!2<E,,|>,,|> 4(  4( r"c  URU5n URU U U UUUUUUUS9 n U Sn URU 5n X-n URU 5n UR U 5nUR U5nX-n U 4U SS-nU$)NrtrCrDrurvrrrr)rrrrrr)r_rmrBrtrCrDrurvrrnormed attn_outputs attn_outputout1out2 ffn_outputoutputss r rzEncoderLayer.forwards#00    !)/)1  #1o mmK0 t$XXd^ ]]:.  'L,,r")rrrrrr)g?Nr|)r}r~rrrSrzrrrs@r rrs& ) ""r"rc*\rSrSr%\\S'SrSrSrg)CTRLPreTrainedModelconfig transformerc$[U[R[45(akURR R SURRS9 URb%URR R5 gg[U[R5(axURR R SURRS9 URb2URR URR5 gg[U[R5(aJURR R5 URR RS5 gg)zInitialize the weights.g)meanstdN?) isinstancerrXrweightdatanormal_rinitializer_rangebiaszero_ Embedding padding_idxrfill_)r_modules r _init_weights!CTRLPreTrainedModel._init_weightss fryy&1 2 2 MM   & &CT[[5R5R & S{{&   &&('  - - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  - - KK   " " $ MM   $ $S ).r"N) r}r~rrr__annotations__base_model_prefixrrrr"r rrs %*r"rc^\rSrSrU4SjrSrSrSr\SS\ \ RS\ \ \ \ RS\ \ RS \ \ RS \ \ RS \ \ RS \ \ RS \ \S\ \S\ \S\ \S\ \ RS\\ \ R\44Sjj5rSrU=r$) CTRLModelc >[TU]U5 URUlURUl[ URUR[R5Ul [R"URUR5Ul[R"UR 5Ul[R$"['UR5Vs/sH8n[)URUR*UR,UR.US9PM: sn5Ul[R2"URUR4S9UlUR95 gs snf)Nrr)rRrSn_embdrn_layer num_layersr5 n_positionsrfloatr4rr vocab_sizewr embd_pdropdropout ModuleListrangern_headr resid_pdrophrlayer_norm_epsilon layernorm post_init)r_rrr`s r rSCTRLModel.__init__s  "MM ../0B0BDDUDUW\WbWbcf//?zz&"3"34 v~~. .AV]]FMM6::vGYGYefg.   fmm9R9RS  s#>E,cUR$rQr)r_s r get_input_embeddingsCTRLModel.get_input_embeddingss vv r"cXlgrQr)r_new_embeddingss r set_input_embeddingsCTRLModel.set_input_embeddingssr"cUR5H-up#URURRU5 M/ g)zf Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} N)itemsrrrh)r_heads_to_prunelayerres r _prune_headsCTRLModel._prune_headss5+002LE FF5M . . : :5 A3r" input_idspast_key_valuesrCtoken_type_ids position_idsrD inputs_embedsrurvoutput_hidden_states return_dictrrreturnc U bU OURRn UbUOURRnU bU OURRn U bU OURRn UbUb [ S5eUbFUR X5 UR5nURSUS5nURSnO1Ub#UR5SSnURSnO [ S5eUb URO URnSnU(aB[U[5(d-[RS5 Sn[R "U5nUbUR#5OSnUc<[$R&"UUSU-[$R(US 9nUR+S5nUbUS::a [ S 5eURUS5nUR+S 5R+S 5nUR-UR.S 9nSU- [$R0"UR.5R2-nUR5X`RR65nUbJURSUS5nUR9U5nU[:R<"UR>5-nOSnUcUR9U5nUSn[$R@"[$RB"UU-UU-5S 5R-U5nU[:R<"UR>5-nURDR-U5Ul"URDUSS24nUU-U-nURGU5nU (aSOSnU (aSOSn[IURJ5H:unnU (aUU4-nU"UUUUUUUU U S9nUSnU (dM1UUS 4- nM< URMU5nU (aUU4-nU(aURO5nU (d[QSUUUU455$[SUUUUS9$)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) Example: ```python >>> from transformers import AutoTokenizer, CTRLModel >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLModel.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 5, 1280] ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer&rz5You have to specify either input_ids or inputs_embedsFzPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `DynamicCache` instead, e.g. `past_key_values=DynamicCache.from_legacy_cache(past_key_values)`.T)r%devicez$batch_size has to be defined and > 0rrr$rrrc3,# UH ocMUv M g7frQr).0rAs r $CTRLModel.forward..s^a^s )last_hidden_stater hidden_states attentions)*rrvruruse_return_dict ValueError%warn_if_padding_and_no_attention_maskr=viewr:rrr logger warning_oncer from_legacy_cacheget_seq_lengthrr)longr,r+r%finfomin get_head_maskrrr;r<rtriuonesr4r enumeraterrto_legacy_cachetupler )r_rrrCrrrDrrurvrrrrkwargs input_shapernrreturn_legacy_cache past_lengthtoken_type_embedsseq_lenrB pos_embedsrall_hidden_statesall_attentionsrrrs r rzCTRLModel.forwards:b2C1N-TXT_T_TqTq!*!6IDKKcd d  "  6 6y Q#..*K!r;r?;I"+J  &',,.s3K&,,Q/JTU U%.%:!! @T@T# Z??   U  #' *<<_MO:I:Uo446[\   << [_{5RZ_ZdZdmstL'11!4L  %Q !GHH+00R@N ,55a8BB1EN,..TZZ.@N!N2ekk$**6M6Q6QQN&&y++2E2EF  %+00[_EN $~ 6  ):):!; ;  !    FF9-Mb/zz%**W{%:Gkg $D>g 'tng d^g !.g  uU\\"$;; <g g r"rz The CTRL Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ) custom_introc ^\rSrSrS/rU4Sjr\SS\\RS\\ \ \RS\\RS\\RS\\RS \\RS \\RS \\RS \\ S \\ S\\ S\\ S\\RS\\ \R\44Sjj5rSSjrSrU=r$)CTRLLMHeadModelizlm_head.weightc>[TU]U5 [U5Ul[R "UR URSS9UlUR5 g)NTr) rRrSrrrrXrrlm_headrr_rr`s r rSCTRLLMHeadModel.__init__sG  $V,yy0A0AM  r"rrrCrrrDrlabelsrurvrrrrrc U bU OURRn URUUUUUUUU U U U U S9 nUSnURU5nSnUb*UR"UU4SURR 0UD6nU (dU4USS-nUbU4U-$U$[ UUURURURS9$)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` Example: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLLMHeadModel >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLLMHeadModel.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Wikipedia The llama is", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> sequence_ids = model.generate(inputs["input_ids"]) >>> sequences = tokenizer.batch_decode(sequence_ids) >>> sequences ['Wikipedia The llama is a member of the family Bovidae. It is native to the Andes of Peru,'] >>> outputs = model(**inputs, labels=inputs["input_ids"]) >>> round(outputs.loss.item(), 2) 9.21 >>> list(outputs.logits.shape) [1, 5, 246534] ```N) rrCrrrDrrurvrrrrrrr)losslogitsrrr) rrrr loss_functionrrrrr)r_rrrCrrrDrrrurvrrrrrtransformer_outputsr lm_logitsrrKs r rzCTRLLMHeadModel.forwards x&1%c $D>c 'tnc d^c !.c uU\\"$:: ;!c c Jddr"ra The CTRL Model transformer with a sequence classification head on top (linear layer). [`CTRLForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). c^\rSrSrU4Sjr\SS\\RS\\ \ \RS\\RS\\RS\\RS\\RS \\RS \\RS \\ S \\ S \\ S\\ S\ \ \R\44Sjj5rSrU=r$)CTRLForSequenceClassificationiCc>[TU]U5 URUl[U5Ul[ R "URURSS9UlUR5 g)NFr) rRrS num_labelsrrrrXr classifierrrs r rS&CTRLForSequenceClassification.__init__OsR   ++$V,))FMM4??O r"rrrCrrrDrrrurvrrrc U bU OURRn URUUUUUUUU U U U S9 n U SnURU5nUbURSSunnOURSSunnURR cUS:wa [ S5eURR cSnOUbXRR :gRUR[R5n[R"URSUR[RS9nUU-RS5nO.Sn[RURR S 35 U[R"UURS 9U4nSnUGbURR"cUR$S:XaS URlOoUR$S:aNUR&[R(:XdUR&[R*:XaS URlOS URlURR"S :XaJ[-5nUR$S:Xa&U"UR/5UR/55nOU"UU5nOURR"S :Xa=[15nU"UR3SUR$5UR3S55nO-URR"S :Xa[55nU"UU5nU (dU4U SS-nUbU4U-$U$[7UUU R8U R:S9$)a" input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Example of single-label classification: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLForSequenceClassification >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> predicted_class_id = logits.argmax().item() >>> model.config.id2label[predicted_class_id] 'LABEL_0' ``` ```python >>> import torch >>> torch.manual_seed(42) # doctest: +IGNORE_RESULT >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` >>> num_labels = len(model.config.id2label) >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl", num_labels=num_labels) >>> labels = torch.tensor(1) >>> loss = model(**inputs, labels=labels).loss >>> round(loss.item(), 2) 0.93 ``` Example of multi-label classification: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLForSequenceClassification >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLForSequenceClassification.from_pretrained( ... "Salesforce/ctrl", problem_type="multi_label_classification" ... ) >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> predicted_class_id = logits.argmax().item() >>> model.config.id2label[predicted_class_id] 'LABEL_0' ``` ```python >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` >>> num_labels = len(model.config.id2label) >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl", num_labels=num_labels) >>> num_labels = len(model.config.id2label) >>> labels = torch.nn.functional.one_hot(torch.tensor([predicted_class_id]), num_classes=num_labels).to( ... torch.float ... ) >>> loss = model(**inputs, labels=labels).loss >>> loss.backward() # doctest: +IGNORE_RESULT ```N) rrCrrrDrrurvrrrrrz=Cannot handle batch sizes > 1 if no padding token is defined.r&)rr%z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`)r regressionsingle_label_classificationmulti_label_classification)rrrr)rrrr,r: pad_token_idrr+rrint32r)argmaxrrr`r} problem_typer+r%rrVrsqueezerrrrrr)r_rrrCrrrDrrrurvrrrrrrnsequence_lengthlast_non_pad_token non_pad_mask token_indices pooled_logitsrloss_fctrKs r rz%CTRLForSequenceClassification.forwardXsP&1%!F!Fr!J !#    >>**+,ZZ  u||Jv}}MOaab   {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#M$9$9$;V^^=MND#M6:D))-JJ+- 2 22t GUWY))-II,. v6#%(;AB(??F)-)9TGf$ Ev E' -;;*55   r")r,r+rr )r}r~rrrSrrrrrrrrrrrzrrrs@r r)r)CsW15EI6:59371559-1$(,0/3&*p E,,-p "%e.?.?(@"ABp !!2!23 p !!1!12 p u//0 p E--.p   1 12p ))*p D>p $D>p 'tnp d^p  uU\\"$<< =p p r"r))r)rrrr&)/__doc__typingrrnumpyr;rrtorch.nnrrr cache_utilsr r generationr modeling_outputsr rrmodeling_utilsr pytorch_utilsrrrutilsrrconfiguration_ctrlr get_loggerr}rr!r5rLModulerNrrrrrr)__all__rr"r rLs" " AA.)ii-YY+   H % %4DDN`/299/d*/***L #L L ^ Ad)?Ad  AdH { $7{  { | cr"