tj/ddlZddlZddlmZddlmZddlZejjj ejjj ejjj ejjj ejjj ejjjejjjejjjejjjejjjejjjejjjejjjejjjdZ d3dejded ed ejdzd ejf d Z d3dejd eded ejdzd ejf dZ dejded ejfdZ dejd ejfdZ dejd ejfdZ dejd ejfdZd4dejded ejfdZd5dejded ejdzd ejfdZd5dejded ejdzd ejfdZ d6dejdededed ejdzd ejf dZ d6dejdededed ejdzd ejf d Z d7dejd ededed ed ejdzd ejfd#Z d8dejded ejdzd ejfd$Z d9dejd&eded ejdzd ejf d'Zdejd(ejd ejfd)Zd:d+Zd,Zd-Z d.Z!ed/Z"ed0Z#ed1Z$ed2Z%dS);N) defaultdict)contextmanager)uniform_normal_ constant_ones_zeros_eye_dirac_xavier_uniform_xavier_normal_kaiming_uniform_kaiming_normal_ trunc_normal_ orthogonal_sparse_?tensorab generatorreturncZt|ddstd||||S|S)N_is_hf_initializedFr)rrrgetattrTORCH_INIT_FUNCTIONS)rrrrs Y/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/initialization.pyrr*s; 6/ 7 7W#J/!qIVVVV MmeanstdcZt|ddstd||||S|S)NrFr)r!r"rr)rr!r"rs rrr2s< 6/ 7 7`#I.vDcU^____ Mr valcVt|ddstd||S|S)NrFr)r$r)rr$s rrr:s5 6/ 7 7B#K0SAAAA Mr cRt|ddstd|S|S)NrFrrrs rrr@s/ 6/ 7 75#G,V444 Mr cRt|ddstd|S|S)NrFr rr's rr r Fs/ 6/ 7 76#H-f555 Mr cRt|ddstd|S|S)NrFr rr's rr r Ls/ 6/ 7 74#F+F333 Mr groupscVt|ddstd||S|S)NrFr )r+r)rr+s rr r Rs5 6/ 7 7E#H-fVDDDD Mr gaincXt|ddstd|||S|S)NrFr r-rrrr-rs rr r Xs9 6/ 7 7_#$56vDT]^^^^ Mr cXt|ddstd|||S|S)NrFr r/rr0s rr r ^s9 6/ 7 7^#$45f4S\]]]] Mr fan_in leaky_relumode nonlinearityc\t|ddstd|||||S|S)NrFrrr4r5rrrrr4r5rs rrrdsH 6/ 7 7 #$67 ad     Mr c\t|ddstd|||||S|S)NrFrr7rr8s rrrrsH 6/ 7 7 #$56 ad     Mr @c^t|ddstd||||||S|S)NrFr)r!r"rrrr)rr!r"rrrs rrrsB 6/ 7 7p#O4V$CSTXYenoooo Mr cXt|ddstd|||S|S)NrFrr/rr0s rrrs: 6/ 7 7[#M26PYZZZZ Mr {Gz?sparsitycZt|ddstd||||S|S)NrFr)r?r"rr)rr?r"rs rrrs< 6/ 7 7h#I.vc]fgggg Mr otherct|dds@tj5||cdddS#1swxYwY|S)NrF)rtorchno_gradcopy_)rrAs rrErEs 6/ 7 7' ]__ ' '<<&& ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' MsAA A normalctjj|\}}|dkr|}n|dkr|}n|dkr||zdz }d|z }|dkr(t |t j|dz dS|d kr%t|t j|dS|d kr+t jd |z}t|| |dStd |) Nr2fan_outfan_avgrtruncated_normalg۶%?)r"rFuniformzinvalid distribution ) rCnninit_calculate_fan_in_and_fan_outrmathsqrtrr ValueError)rr4 distributionr2rHdenomvariancebounds r_variance_scalingrXshmAA&IIOFG x      '!Q&U{H)))f$)H"5"58K"KLLLLLL  ! !DIh//000000  " " !h,''%'''''???@@@r cLt|ddst|dd|S)NrFr2rKr4rTrrXr's r lecun_normal_r\s3 6/ 7 7R&x>PQQQQ Mr cLt|ddst|dd|S)NrFr2rFrZr[r's rdefault_flax_embed_init_r^s2 6/ 7 7H&xhGGGG Mr ) z torch.nn.initztorch.nn.modules.activationztorch.nn.modules.transformerztorch.nn.modules.linearztorch.nn.modules.lossztorch.nn.modules.batchnormztorch.nn.modules.convztorch.nn.modules.normalizationztorch.nn.modules.rnnztorch.nn.modules.sparsec #~Ktt} tD]}|tjvrztj|}t D]N}t||rvy7Q7Q &))4 799Y3GHHH "+!2!2 1 1 FI#,??#4#4 1 1 4 40000 1 1 1!2!2 1 1 FI#,??#4#4 1 1 4 40000 1 1s BC55AD<c #Kddlm}d}tt} tD]x}|t jvrht j|}tD]<}t||r*t|||||<t|||=y|j }||_ dV| D]0\}}| D]\}}t|||1||_ dS#| D]0\}}| D]\}}t|||1||_ wxYw)ac Disable weight initialization both at the torch-level, and at the transformers-level (`init_weights`). This is used to speed-up initializing an empty model with deepspeed, as we do not initialize the model on meta device with deepspeed, but we still don't need to run expensive weight initializations as we are loading params afterwards. r*PreTrainedModelcdSNargskwargss r empty_funcz#no_init_weights..empty_func r N)modeling_utilsrrrr`rarbrcrrdrerrf init_weightsrh) rrryrirjrkrloriginal_init_weightsrmrns rno_init_weightsr~s0/////   D!!I=1 ? ?Kck))[1!5!:!:!vy7Q7Q &))4 :>>>!0 <'1$ "+!2!2 1 1 FI#,??#4#4 1 1 4 40000 1(=$$$ "+!2!2 1 1 FI#,??#4#4 1 1 4 40000 1(=$<<<.empty_func$rzr N)r{rr tie_weights)rrryoriginal_tie_weightss rno_tie_weightsrsm0/////   ;.:&0# ';###&:#::::s( 1c#Ktjfd}|t_ dVt_dS#t_wxYw)a During meta-device model initialisation, ``torch.linspace`` produces meta tensors that have no data. Custom models loaded from the Hub (remote code) often call ``.item()`` on these tensors to compute scalar hyperparameters (e.g. stochastic-depth / drop-path schedules). Native transformers models already pass ``device="cpu"`` explicitly for such calls (see e.g. ``modeling_swin.py``, ``modeling_pvt_v2.py``), but remote-code models written before v5 do not. This context manager patches ``torch.linspace`` to default to ``device="cpu"`` when no explicit device is requested, matching the best practice already used throughout transformers. Calls that supply an explicit ``device`` argument (e.g. ``device=self.logits.device``) are left untouched. ``torch.arange`` is intentionally NOT patched because it is used in RoPE computations where the device must match model parameters. c@|dd|i|S)Ndevicecpu) setdefault)rwrxoriginal_linspaces r_safe_linspacez5meta_device_safe_creation_ops.._safe_linspaceEs/(E***  $1&111r N)rClinspace)rrs @rmeta_device_safe_creation_opsr1s`$22222$EN+ ******s 4A)rrN)r*)rN)rr2r3N)rrr:r;N)r*N)r>N)r2rF)&rQrb collectionsr contextlibrrCrNrOrrrrr r r r r rrrrrrTensorfloat GeneratorintstrrErXr\r^raror~rrrur rrs ######%%%%%%  &x}$( X] hm" HM hm"x}4hm2 6x}4X]08=,x}$&_c L"-2EJ_W[E[ \dh L %27JO/\`J` \el5< %,5< 5<EL %, 5<U\ ELZ^H^jojv 5<uuY]G]iniu$(,   L       %   \     $(,   L       %   \     (,   L         %  \    (, L % \ cg L$)05IN[_I_ \%,u| AAAA,  1114!=!=!=H;;;*+++++r