o sh-@s$ddlZddlmZddlZddlmZmZddlmZddlm Z e e Z Gdddej ZGd d d ej ZGd d d ej ZGd ddej ZGdddej ZGdddej ZGdddej ZGdddej ZGdddej ZGdddej ZGdddej ZGdddeZGdd d ej Zid!ed"ed#d$d%fd&ed'ed(ed)d*ifd+ed,ed-ed.ejd/ed0ed1ed2ejd3ed4ejd5ejd6ejejej ej!ed7Z"ee"Z#d8d9Z$e$d(Z%e$d'Z&e$d!Z'e$d&Z(e$d1Z)e$d6Z*e$d0Z+e$d/Z,dS):N) OrderedDict)Tensornn)logging)is_torchdynamo_compilingc@"eZdZdZdedefddZdS)PytorchGELUTanha& A fast C implementation of the tanh approximation of the GeLU activation function. See https://huggingface.co/papers/1606.08415. This implementation is equivalent to NewGELU and FastGELU but much faster. However, it is not an exact numerical match due to rounding errors. inputreturncCstjj|ddS)Ntanh) approximate)r functionalgeluselfr rV/var/www/html/alpaca_bot/venv/lib/python3.10/site-packages/transformers/activations.pyforward%szPytorchGELUTanh.forwardN__name__ __module__ __qualname____doc__rrrrrrr sr c@r)NewGELUActivationz Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see the Gaussian Error Linear Units paper: https://huggingface.co/papers/1606.08415 r r c Cs6d|dttdtj|dt|dS)N??@Hm?g@)torchr mathsqrtpipowrrrrr/s6zNewGELUActivation.forwardNrrrrrr)srcsLeZdZdZd deffdd Zdedefdd Zdedefd d ZZ S) GELUActivationa Original Implementation of the GELU activation function in Google BERT repo when initially created. For information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional Also see the Gaussian Error Linear Units paper: https://huggingface.co/papers/1606.08415 Fuse_gelu_pythoncs(t|r |j|_dStjj|_dSN)super__init__ _gelu_pythonactrrr)rr% __class__rrr(;s  zGELUActivation.__init__r r cCs |ddt|tdS)Nrrr)rerfr r!rrrrr)Bs zGELUActivation._gelu_pythoncC ||Sr&r*rrrrrE zGELUActivation.forward)F) rrrrboolr(rr)r __classcell__rrr+rr$3s r$c@r)FastGELUActivationz} Applies GELU approximation that is slower than QuickGELU but more accurate. See: https://github.com/hendrycks/GELUs r r cCs*d|dt|ddd||S)Nrrg3E?r)rr rrrrrNs*zFastGELUActivation.forwardNrrrrrr3Ir3c@r)QuickGELUActivationzr Applies GELU approximation that is fast but somewhat inaccurate. See: https://github.com/hendrycks/GELUs r r cCs|td|S)NgZd;?)rsigmoidrrrrrWszQuickGELUActivation.forwardNrrrrrr5Rr4r5cs<eZdZdZdedeffdd Zdedefdd ZZS) ClippedGELUActivationa Clip the range of possible GeLU outputs between [min, max]. This is especially useful for quantization purpose, as it allows mapping negatives values in the GeLU spectrum. For more information on this trick, please refer to https://huggingface.co/papers/2004.09602. Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))). See https://huggingface.co/papers/1606.08415 minmaxcs8||krtd|d|dt||_||_dS)Nzmin should be < max (got min: z, max: )) ValueErrorr'r(r8r9)rr8r9r+rrr(hs   zClippedGELUActivation.__init__xr cCstt||j|jSr&)rcliprr8r9)rr<rrrrpzClippedGELUActivation.forward) rrrrfloatr(rrr2rrr+rr7[s r7cs2eZdZdZfddZdedefddZZS)AccurateGELUActivationz Applies GELU approximation that is faster than default and more accurate than QuickGELU. See: https://github.com/hendrycks/GELUs Implemented along with MEGA (Moving Average Equipped Gated Attention) cs ttdtj|_dS)N)r'r(r r!r"precomputed_constantrr+rrr(|s zAccurateGELUActivation.__init__r r c Cs,d|dt|j|dt|dS)Nrrr)rr rBr#rrrrrs,zAccurateGELUActivation.forward)rrrrr(rrr2rrr+rr@ts r@csDeZdZdZfddZdedefddZdedefdd ZZS) MishActivationz See Mish: A Self-Regularized Non-Monotonic Activation Function (Misra., https://huggingface.co/papers/1908.08681). Also visit the official repository for the paper: https://github.com/digantamisra98/Mish csttjj|_dSr&)r'r(rrmishr*rCr+rrr(s zMishActivation.__init__r r cCs|ttj|Sr&)rr rrsoftplusrrrr _mish_pythonr>zMishActivation._mish_pythoncCr.r&r/rrrrrr0zMishActivation.forward) rrrrr(rrHrr2rrr+rrEs  rEc@r)LinearActivationz[ Applies the linear activation function, i.e. forwarding input directly to output. r r cCs|Sr&rrrrrrszLinearActivation.forwardNrrrrrrIr4rIc@seZdZdZdddZdS)LaplaceActivationz Applies elementwise activation based on Laplace function, introduced in MEGA as an attention activation. See https://huggingface.co/papers/2209.10655 Inspired by squared relu, but with bounded range and gradient for better stability 绹۞? ^/ ?cCs*|||td}ddt|S)Nrrr)divr r!rr-)rr musigmarrrrszLaplaceActivation.forwardN)rKrLrrrrrrrrrrJsrJc@seZdZdZddZdS)ReLUSquaredActivationz` Applies the relu^2 activation introduced in https://huggingface.co/papers/2109.08668v2 cCstj|}t|}|Sr&)rrrelursquare)rr relu_appliedsquaredrrrrs  zReLUSquaredActivation.forwardNrPrrrrrQs rQcseZdZfddZZS)ClassInstantiercs4t|}t|tr |n|if\}}|di|S)Nr)r' __getitem__ isinstancetuple)rkeycontentclskwargsr+rrrWs zClassInstantier.__getitem__)rrrrWr2rrr+rrVsrVcsfeZdZdZddddejdffdd Zded efd d Zded efd d Z ded efddZ Z S)XIELUActivationz Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010 If the user has installed the nickjbrowning/XIELU wheel, we import xIELU CUDA Otherwise, we emit a single warning and use xIELU Python g?rgưFc stttttj||ddd|_ ttttj|||ddd|_ | dtj||d| dtj||d||_ t |j |_t |j |_d|_zFddl}tjj|_d}zddlm} | |j|_|d7}Wnty} z|d | d 7}|j|_WYd} ~ nd} ~ wwt |WdSty} zt d t!| WYd} ~ dSd} ~ ww) N)dtyperrbetaepszUsing experimental xIELU CUDA.)allow_in_graphz& Enabled torch._dynamo for xIELU CUDA.z+ Could not enable torch._dynamo for xIELU (z*) - this may result in slower performance.uCUDA-fused xIELU not available (%s) – falling back to a Python version. For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`)"r'r(r Parameterrlogexptensor unsqueezealpha_palpha_nregister_bufferwith_vector_loadsr?r`detachcpuitem _beta_scalarra _eps_scalar_xielu_cuda_obj xielu.opsclassesxieluXIELU torch._dynamorb _xielu_cuda_xielu_cuda_fn Exceptionlogger warning_oncestr) r alpha_p_init alpha_n_initr`rar_rkrtmsgrberrr+rrr(s@ ,&   zXIELUActivation.__init__r<r c Cshtj|j}|jtj|j}t|dk||||j|tt ||j |||j|S)Nr) rrrGrhr`rirwhereexpm1r8ra)rr<rhrirrr _xielu_pythons$zXIELUActivation._xielu_pythoncCs|j}|dkr|d}|dks |dkr$|dd|d}||jkr1td||j|j||j |j |j |j |j }||S)zDFirewall function to prevent torch.compile from seeing .item() callsrDrrz_Warning: xIELU input tensor expects 3 dimensions but got (shape: %s). Reshaping to (shape: %s).)shapedimrgviewsizerzr{rqrrhrirorprk)rr<original_shaperesultrrrrws*     zXIELUActivation._xielu_cudar cCs4|jdur|jrts||Std||S)Nz:torch._dynamo is compiling, using Python version of xIELU.)rqis_cudarrxrzr{rrrrrrs    zXIELUActivation.forward) rrrrrbfloat16r(rrrwrr2rrr+rr^s + r^rgelu_10i )r8r9 gelu_fastgelu_new gelu_pythonr%Tgelu_pytorch_tanh gelu_accuratelaplace leaky_relulinearrF quick_gelurRrelu2relu6r6silu)swishr prelurtcCs,|tvrt|Std|dtt)Nz function z not found in ACT2FN mapping )ACT2FNKeyErrorlistkeys)activation_stringrrrget_activation5sr)-r collectionsrrrrutilsrutils.import_utilsr get_loggerrrzModuler rr$r3r5r7r@rErIrJrQrVr^ LeakyReLUReLUReLU6SigmoidSiLUTanhPReLUACT2CLSrrrrrrrrrF linear_actrrrrs           ^