
    h               	          d Z ddlZddlmZ ddlmZmZ ddlZddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ  e       r	ddlmZmZ nd Zd Z ej@                  e!      Z"e ed       G d de                    Z#e ed       G d de                    Z$e ed       G d de                    Z% G d dejL                        Z' G d dejL                        Z( G d  d!ejL                        Z)dDd"ejT                  d#e+d$e,d%ejT                  fd&Z- G d' d(ejL                        Z. G d) d*ejL                        Z/ G d+ d,ejL                        Z0 G d- d.ejL                        Z1 G d/ d0ejL                        Z2 G d1 d2ejL                        Z3 G d3 d4ejL                        Z4 G d5 d6ejL                        Z5 G d7 d8ejL                        Z6e G d9 d:e             Z7e G d; d<e7             Z8 ed=       G d> d?e7             Z9 ed@       G dA dBe7e             Z:g dCZ;y)Ez9PyTorch Dilated Neighborhood Attention Transformer model.    N)	dataclass)OptionalUnion)nn   )ACT2FN)BackboneOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)ModelOutputOptionalDependencyNotAvailableauto_docstringis_natten_availableloggingrequires_backends)BackboneMixin   )DinatConfig)
natten2davnatten2dqkrpbc                      t               Nr   argskwargss     g/var/www/html/aiagenthome/venv/lib/python3.12/site-packages/transformers/models/dinat/modeling_dinat.pyr   r   ,       ,..    c                      t               r   r   r   s     r   r   r   /   r   r    zO
    Dinat encoder's outputs, with potential hidden states and attentions.
    )custom_introc                       e Zd ZU dZdZeej                     ed<   dZ	ee
ej                  df      ed<   dZee
ej                  df      ed<   dZee
ej                  df      ed<   y)DinatEncoderOutputa  
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlast_hidden_state.hidden_states
attentionsreshaped_hidden_states)__name__
__module____qualname____doc__r%   r   torchFloatTensor__annotations__r&   tupler'   r(    r    r   r$   r$   9   s}     6:x 1 129=AM8E%"3"3S"89:A:>Ju00#567>FJHU5+<+<c+A%BCJr    r$   zW
    Dinat model's outputs that also contains a pooling of the last hidden states.
    c                       e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   dZeeej                  df      ed<   dZeeej                  df      ed<   y)	DinatModelOutputa  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
        Average pooling of the last layer hidden-state.
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nr%   pooler_output.r&   r'   r(   )r)   r*   r+   r,   r%   r   r-   r.   r/   r4   r&   r0   r'   r(   r1   r    r   r3   r3   O   s    	 6:x 1 12915M8E--.5=AM8E%"3"3S"89:A:>Ju00#567>FJHU5+<+<c+A%BCJr    r3   z1
    Dinat outputs for image classification.
    c                       e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   dZeeej                  df      ed<   dZeeej                  df      ed<   y)	DinatImageClassifierOutputa7  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, hidden_size, height, width)`.

        Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
        include the spatial dimensions.
    Nlosslogits.r&   r'   r(   )r)   r*   r+   r,   r7   r   r-   r.   r/   r8   r&   r0   r'   r(   r1   r    r   r6   r6   h   s     )-D(5$$
%,*.FHU&&'.=AM8E%"3"3S"89:A:>Ju00#567>FJHU5+<+<c+A%BCJr    r6   c                   f     e Zd ZdZ fdZdeej                     deej                     fdZ
 xZS )DinatEmbeddingsz6
    Construct the patch and position embeddings.
    c                     t         |           t        |      | _        t	        j
                  |j                        | _        t	        j                  |j                        | _
        y r   )super__init__DinatPatchEmbeddingspatch_embeddingsr   	LayerNorm	embed_dimnormDropouthidden_dropout_probdropoutselfconfig	__class__s     r   r=   zDinatEmbeddings.__init__   sG     4V <LL!1!12	zz&"<"<=r    pixel_valuesreturnc                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r?   rB   rE   )rG   rJ   
embeddingss      r   forwardzDinatEmbeddings.forward   s4    **<8
YYz*
\\*-
r    )r)   r*   r+   r,   r=   r   r-   r.   r0   TensorrN   __classcell__rI   s   @r   r:   r:      s4    >HU->->$? E%,,DW r    r:   c                   `     e Zd ZdZ fdZdeej                     dej                  fdZ	 xZ
S )r>   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, height, width, hidden_size)` to be consumed by a
    Transformer.
    c           
      P   t         |           |j                  }|j                  |j                  }}|| _        |dk(  rnt        d      t        j                  t        j                  | j                  |dz  ddd      t        j                  |dz  |ddd            | _	        y )N   z2Dinat only supports patch size of 4 at the moment.   r   r   rU   rU   r   r   )kernel_sizestridepadding)
r<   r=   
patch_sizenum_channelsrA   
ValueErrorr   
SequentialConv2d
projection)rG   rH   r\   r]   hidden_sizerI   s        r   r=   zDinatPatchEmbeddings.__init__   s    &&
$*$7$79I9Ik(? QRR--IId'')9vV\flmIIkQ&PV`fg
r    rJ   rK   c                     |j                   \  }}}}|| j                  k7  rt        d      | j                  |      }|j	                  dddd      }|S )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r   rU   r   r   )shaper]   r^   ra   permute)rG   rJ   _r]   heightwidthrM   s          r   rN   zDinatPatchEmbeddings.forward   s`    )5););&<4,,,w  __\2
''1a3
r    )r)   r*   r+   r,   r=   r   r-   r.   rO   rN   rP   rQ   s   @r   r>   r>      s/    
"	HU->->$? 	ELL 	r    r>   c                        e Zd ZdZej
                  fdedej                  ddf fdZde	j                  de	j                  fdZ xZS )	DinatDownsamplerz
    Convolutional Downsampling Layer.

    Args:
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    dim
norm_layerrK   Nc                     t         |           || _        t        j                  |d|z  dddd      | _         |d|z        | _        y )NrU   rV   rW   rX   F)rY   rZ   r[   bias)r<   r=   rk   r   r`   	reductionrB   )rG   rk   rl   rI   s      r   r=   zDinatDownsampler.__init__   sE    3CVF\binoq3w'	r    input_featurec                     | j                  |j                  dddd            j                  dddd      }| j                  |      }|S )Nr   r   r   rU   )ro   re   rB   )rG   rp   s     r   rN   zDinatDownsampler.forward   sJ    }'<'<Q1a'HIQQRSUVXY[\]		-0r    )r)   r*   r+   r,   r   r@   intModuler=   r-   rO   rN   rP   rQ   s   @r   rj   rj      sJ     :< (C (RYY ($ (U\\ ell r    rj   input	drop_probtrainingrK   c                    |dk(  s|s| S d|z
  }| j                   d   fd| j                  dz
  z  z   }|t        j                  || j                  | j
                        z   }|j                          | j                  |      |z  }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
            r   r   )r   )dtypedevice)rd   ndimr-   randry   rz   floor_div)rt   ru   rv   	keep_probrd   random_tensoroutputs          r   	drop_pathr      s     CxII[[^

Q 77E

5ELL YYMYYy!M1FMr    c                   x     e Zd ZdZd	dee   ddf fdZdej                  dej                  fdZ	de
fdZ xZS )
DinatDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nru   rK   c                 0    t         |           || _        y r   )r<   r=   ru   )rG   ru   rI   s     r   r=   zDinatDropPath.__init__   s    "r    r&   c                 D    t        || j                  | j                        S r   )r   ru   rv   rG   r&   s     r   rN   zDinatDropPath.forward   s    FFr    c                      d| j                    S )Nzp=)ru   rG   s    r   
extra_reprzDinatDropPath.extra_repr   s    DNN#$$r    r   )r)   r*   r+   r,   r   floatr=   r-   rO   rN   strr   rP   rQ   s   @r   r   r      sG    b#(5/ #T #GU\\ Gell G%C %r    r   c                   j     e Zd Z fdZ	 ddej
                  dee   deej
                     fdZ	 xZ
S )NeighborhoodAttentionc                 *   t         |           ||z  dk7  rt        d| d| d      || _        t	        ||z        | _        | j                  | j
                  z  | _        || _        || _        t        j                  t        j                  |d| j                  z  dz
  d| j                  z  dz
              | _        t        j                  | j                  | j                  |j                        | _        t        j                  | j                  | j                  |j                        | _        t        j                  | j                  | j                  |j                        | _        t        j&                  |j(                        | _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()rU   r   )rn   )r<   r=   r^   num_attention_headsrr   attention_head_sizeall_head_sizerY   dilationr   	Parameterr-   zerosrpbLinearqkv_biasquerykeyvaluerC   attention_probs_dropout_probrE   rG   rH   rk   	num_headsrY   r   rI   s         r   r=   zNeighborhoodAttention.__init__   sD   ?a#C5(^_h^iijk  $- #&sY#7 !558P8PP&  <<ID<L<L8Lq8PTUX\XhXhThklTl noYYt1143E3EFOO\
99T//1C1C&//ZYYt1143E3EFOO\
zz&"E"EFr    r&   output_attentionsrK   c                    |j                   \  }}}| j                  |      j                  |d| j                  | j                        j                  dd      }| j                  |      j                  |d| j                  | j                        j                  dd      }| j                  |      j                  |d| j                  | j                        j                  dd      }|t        j                  | j                        z  }t        ||| j                  | j                  | j                        }	t        j                  j!                  |	d      }
| j#                  |
      }
t%        |
|| j                  | j                        }|j'                  ddddd      j)                         }|j+                         d d | j,                  fz   }|j                  |      }|r||
f}|S |f}|S )	Nr   rU   rk   r   r   rT   )rd   r   viewr   r   	transposer   r   mathsqrtr   r   rY   r   r   
functionalsoftmaxrE   r   re   
contiguoussizer   )rG   r&   r   
batch_size
seq_lengthrf   query_layer	key_layervalue_layerattention_scoresattention_probscontext_layernew_context_layer_shapeoutputss                 r   rN   zNeighborhoodAttention.forward  s   
 %2$7$7!
JJJ}%T*b$":":D<T<TUYq!_ 	 HH]#T*b$":":D<T<TUYq!_ 	 JJ}%T*b$":":D<T<TUYq!_ 	 "DIId.F.F$GG )i4K[K[]a]j]jk --//0@b/I ,,7"?KAQAQSWS`S`a%--aAq!<GGI"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2 O\M]r    Fr)   r*   r+   r=   r-   rO   r   boolr0   rN   rP   rQ   s   @r   r   r      s@    G2 -2,||, $D>, 
u||		,r    r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )NeighborhoodAttentionOutputc                     t         |           t        j                  ||      | _        t        j
                  |j                        | _        y r   )r<   r=   r   r   denserC   r   rE   rG   rH   rk   rI   s      r   r=   z$NeighborhoodAttentionOutput.__init__?  s6    YYsC(
zz&"E"EFr    r&   input_tensorrK   c                 J    | j                  |      }| j                  |      }|S r   r   rE   )rG   r&   r   s      r   rN   z#NeighborhoodAttentionOutput.forwardD  s$    

=1]3r    r)   r*   r+   r=   r-   rO   rN   rP   rQ   s   @r   r   r   >  s2    G
U\\  RWR^R^ r    r   c                   p     e Zd Z fdZd Z	 ddej                  dee   de	ej                     fdZ
 xZS )NeighborhoodAttentionModulec                     t         |           t        |||||      | _        t	        ||      | _        t               | _        y r   )r<   r=   r   rG   r   r   setpruned_headsr   s         r   r=   z$NeighborhoodAttentionModule.__init__L  s:    )&#y+xX	1&#>Er    c                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   r   )lenr   rG   r   r   r   r   r   r   r   r   r   r   union)rG   headsindexs      r   prune_headsz'NeighborhoodAttentionModule.prune_headsR  s   u:?749900$))2O2OQUQbQb
u
 -TYY__eD		*499==%@		,TYY__eD		.t{{/@/@%QO )-		(E(EE
(R		%"&))"?"?$))B_B_"_		 --33E:r    r&   r   rK   c                 f    | j                  ||      }| j                  |d   |      }|f|dd  z   }|S Nr   r   )rG   r   )rG   r&   r   self_outputsattention_outputr   s         r   rN   z#NeighborhoodAttentionModule.forwardd  sC    
 yy0AB;;|AF#%QR(88r    r   )r)   r*   r+   r=   r   r-   rO   r   r   r0   rN   rP   rQ   s   @r   r   r   K  sD    ";* -2|| $D> 
u||		r    r   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )DinatIntermediatec                    t         |           t        j                  |t	        |j
                  |z              | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )r<   r=   r   r   rr   	mlp_ratior   
isinstance
hidden_actr   r   intermediate_act_fnr   s      r   r=   zDinatIntermediate.__init__p  sa    YYsC(8(83(>$?@
f''-'-f.?.?'@D$'-'8'8D$r    r&   rK   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r   s     r   rN   zDinatIntermediate.forwardx  s&    

=100?r    r   rQ   s   @r   r   r   o  s#    9U\\ ell r    r   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )DinatOutputc                     t         |           t        j                  t	        |j
                  |z        |      | _        t        j                  |j                        | _	        y r   )
r<   r=   r   r   rr   r   r   rC   rD   rE   r   s      r   r=   zDinatOutput.__init__  sF    YYs6#3#3c#9:C@
zz&"<"<=r    r&   rK   c                 J    | j                  |      }| j                  |      }|S r   r   r   s     r   rN   zDinatOutput.forward  s$    

=1]3r    r   rQ   s   @r   r   r   ~  s#    >
U\\ ell r    r   c            	            e Zd Zd fd	Zd Z	 ddej                  dee   de	ej                  ej                  f   fdZ
 xZS )	
DinatLayerc                    t         |           |j                  | _        |j                  | _        || _        | j                  | j                  z  | _        t        j                  ||j                        | _	        t        |||| j                  | j                        | _        |dkD  rt        |      nt        j                         | _        t        j                  ||j                        | _        t!        ||      | _        t%        ||      | _        |j(                  dkD  r?t        j*                  |j(                  t-        j.                  d|f      z  d      | _        y d | _        y )Neps)rY   r   rx   r   rU   T)requires_grad)r<   r=   chunk_size_feed_forwardrY   r   window_sizer   r@   layer_norm_epslayernorm_beforer   	attentionr   Identityr   layernorm_afterr   intermediater   r   layer_scale_init_valuer   r-   oneslayer_scale_parameters)rG   rH   rk   r   r   drop_path_raterI   s         r   r=   zDinatLayer.__init__  s(   '-'E'E$!-- ++dmm; "Sf6K6K L4C0@0@4==
 ;I3:N~6TVT_T_Ta!||CV5J5JK-fc:!&#. ,,q0 LL66QH9MM]ab 	#  	#r    c                     | j                   }d}||k  s||k  rJdx}}t        d||z
        }t        d||z
        }	dd||||	f}t        j                  j	                  ||      }||fS )N)r   r   r   r   r   r   r   )r   maxr   r   pad)
rG   r&   rg   rh   r   
pad_valuespad_lpad_tpad_rpad_bs
             r   	maybe_padzDinatLayer.maybe_pad  s    &&'
K5;#6EE;./E;/0EQueU;JMM--mZHMj((r    r&   r   rK   c                    |j                         \  }}}}|}| j                  |      }| j                  |||      \  }}|j                  \  }	}
}}	| j	                  ||      }|d   }|d   dkD  xs |d   dkD  }|r|d d d |d |d d f   j                         }| j                  | j                  d   |z  }|| j                  |      z   }| j                  |      }| j                  | j                  |            }| j                  | j                  d   |z  }|| j                  |      z   }|r	||d   f}|S |f}|S )N)r   r   r      r   )r   r   r   rd   r   r   r   r   r   r   r   )rG   r&   r   r   rg   rh   channelsshortcutr   rf   
height_pad	width_padattention_outputsr   
was_paddedlayer_outputlayer_outputss                    r   rN   zDinatLayer.forward  s|   
 /<.@.@.B+
FE8 --m<$(NN=&%$P!z&3&9&9#:y! NN=L]N^,Q/]Q&;*Q-!*;
/7F7FUFA0EFQQS&&2#::1=@PP 4>>2B#CC++M:{{4#4#4\#BC&&266q9LHL$t~~l'CC@Q'8';< YeWfr    )rx   r   )r)   r*   r+   r=   r   r-   rO   r   r   r0   rN   rP   rQ   s   @r   r   r     sM    
(	) -2$||$ $D>$ 
u||U\\)	*	$r    r   c                   j     e Zd Z fdZ	 ddej
                  dee   deej
                     fdZ	 xZ
S )
DinatStagec                 <   t         	|           || _        || _        t	        j
                  t        |      D cg c]  }t        |||||   ||          c}      | _        |% ||t        j                        | _
        d| _        y d | _
        d| _        y c c}w )N)rH   rk   r   r   r   )rk   rl   F)r<   r=   rH   rk   r   
ModuleListranger   layersr@   
downsamplepointing)
rG   rH   rk   depthr   	dilationsr   r	  irI   s
            r   r=   zDinatStage.__init__  s    mm u	 &A !'&q\#1!#4 &	
 !(SR\\JDO  #DO%	s   Br&   r   rK   c                     |j                         \  }}}}t        | j                        D ]  \  }} |||      }|d   } |}	| j                  | j                  |	      }||	f}
|r|
dd  z  }
|
S r   )r   	enumerater  r	  )rG   r&   r   rf   rg   rh   r  layer_moduler  !hidden_states_before_downsamplingstage_outputss              r   rN   zDinatStage.forward  s    
 ,00265!(5OA|(8IJM)!,M  6 -:)??& OO,MNM&(IJ]12..Mr    r   r   rQ   s   @r   r  r    s?    8 -2|| $D> 
u||		r    r  c                   ~     e Zd Z fdZ	 	 	 	 d	dej
                  dee   dee   dee   dee   dee	e
f   fdZ xZS )
DinatEncoderc                    t         |           t        |j                        | _        || _        t        j                  d|j                  t        |j                        d      D cg c]  }|j                          }}t        j                  t        | j                        D cg c]  }t        |t        |j                   d|z  z        |j                  |   |j"                  |   |j$                  |   |t        |j                  d |       t        |j                  d |dz           || j                  dz
  k  rt&        nd        c}      | _        y c c}w c c}w )Nr   cpu)rz   rU   r   )rH   rk   r  r   r  r   r	  )r<   r=   r   depths
num_levelsrH   r-   linspacer   sumitemr   r  r  r  rr   rA   r   r  rj   levels)rG   rH   xdpri_layerrI   s        r   r=   zDinatEncoder.__init__  s6   fmm,!&63H3H#fmmJ\ej!kl!kAqvvx!klmm  %T__5  6G !F,,q'z9: --0$..w7$..w7#&s6=='+B'Cc&--XeZadeZeJfFg#h4;dooPQ>Q4Q/X\  6
 ms   )E(B#Er&   r   output_hidden_states(output_hidden_states_before_downsamplingreturn_dictrK   c                    |rdnd }|rdnd }|rdnd }|r |j                  dddd      }	||fz  }||	fz  }t        | j                        D ]l  \  }
} |||      }|d   }|d   }|r#|r!|j                  dddd      }	||fz  }||	fz  }n$|r"|s |j                  dddd      }	||fz  }||	fz  }|se||dd  z  }n |st        d |||fD              S t	        ||||      S )Nr1   r   r   r   rU   c              3   &   K   | ]	  }||  y wr   r1   ).0vs     r   	<genexpr>z'DinatEncoder.forward.<locals>.<genexpr><  s     m$[q_`_l$[s   )r%   r&   r'   r(   )re   r  r  r0   r$   )rG   r&   r   r   r!  r"  all_hidden_statesall_reshaped_hidden_statesall_self_attentionsreshaped_hidden_stater  r  r  r  s                 r   rN   zDinatEncoder.forward  sX    #7BD+?RT"$5b4$1$9$9!Q1$E!-!11&+@*BB&(5OA|(8IJM)!,M0=a0@-#(P(I(Q(QRSUVXY[\(]%!&G%II!*/D.FF*%.V(5(=(=aAq(I%!m%55!*/D.FF* #}QR'88#%  6( m]4EGZ$[mmm!++*#=	
 	
r    )FFFT)r)   r*   r+   r=   r-   rO   r   r   r   r0   r$   rN   rP   rQ   s   @r   r  r     st    
. -2/4CH&*.
||.
 $D>.
 'tn	.

 3;4..
 d^.
 
u((	).
r    r  c                   &    e Zd ZU eed<   dZdZd Zy)DinatPreTrainedModelrH   dinatrJ   c                    t        |t        j                  t        j                  f      rm|j                  j
                  j                  d| j                  j                         |j                  %|j                  j
                  j                          yyt        |t        j                        rJ|j                  j
                  j                          |j                  j
                  j                  d       yy)zInitialize the weightsrx   )meanstdNg      ?)r   r   r   r`   weightdatanormal_rH   initializer_rangern   zero_r@   fill_)rG   modules     r   _init_weightsz"DinatPreTrainedModel._init_weightsL  s    fryy"))45 MM&&CT[[5R5R&S{{&  &&( '-KK""$MM$$S) .r    N)r)   r*   r+   r   r/   base_model_prefixmain_input_namer9  r1   r    r   r-  r-  F  s    $O
*r    r-  c                        e Zd Zd
 fd	Zd Zd Ze	 	 	 	 ddeej                     dee
   dee
   dee
   deeef   f
d	       Z xZS )
DinatModelc                    t         |   |       t        | dg       || _        t	        |j
                        | _        t        |j                  d| j                  dz
  z  z        | _	        t        |      | _        t        |      | _        t        j                  | j                  |j                         | _        |rt        j$                  d      nd| _        | j)                          y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        nattenrU   r   r   N)r<   r=   r   rH   r   r  r  rr   rA   num_featuresr:   rM   r  encoderr   r@   r   	layernormAdaptiveAvgPool1dpooler	post_init)rG   rH   add_pooling_layerrI   s      r   r=   zDinatModel.__init__[  s    
 	 $
+fmm, 0 0119L3M MN)&1#F+d&7&7V=R=RS1Bb**1- 	r    c                 .    | j                   j                  S r   rM   r?   r   s    r   get_input_embeddingszDinatModel.get_input_embeddingsq      ///r    c                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsrA  layerr   r   )rG   heads_to_prunerM  r   s       r   _prune_headszDinatModel._prune_headst  s>    
 +002LE5LLu%//;;EB 3r    rJ   r   r   r"  rK   c                 R   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      | j                  |      }| j                  ||||      }|d   }| j                  |      }d }| j                  G| j                  |j                  dd      j                  dd            }t        j                  |d      }|s||f|dd  z   }	|	S t        |||j                  |j                  |j                        S )Nz You have to specify pixel_valuesr   r   r"  r   r   rU   )r%   r4   r&   r'   r(   )rH   r   r   use_return_dictr^   rM   rA  rB  rD  flattenr   r-   r3   r&   r'   r(   )
rG   rJ   r   r   r"  embedding_outputencoder_outputssequence_outputpooled_outputr   s
             r   rN   zDinatModel.forward|  sA    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@??<8,,/!5#	 ' 
 *!,..9;;" KK(?(?1(E(O(OPQST(UVM!MM-;M%}58KKFM-')77&11#2#I#I
 	
r    )T)NNNN)r)   r*   r+   r=   rI  rO  r   r   r-   r.   r   r   r0   r3   rN   rP   rQ   s   @r   r=  r=  Y  s    ,0C  59,0/3&*,
u001,
 $D>,
 'tn	,

 d^,
 
u&&	',
 ,
r    r=  z
    Dinat Model transformer with an image classification head on top (a linear layer on top of the final hidden state
    of the [CLS] token) e.g. for ImageNet.
    c                        e Zd Z fdZe	 	 	 	 	 d	deej                     deej                     dee	   dee	   dee	   de
eef   fd       Z xZS )
DinatForImageClassificationc                 X   t         |   |       t        | dg       |j                  | _        t	        |      | _        |j                  dkD  r4t        j                  | j
                  j                  |j                        nt        j                         | _
        | j                          y )Nr?  r   )r<   r=   r   
num_labelsr=  r.  r   r   r@  r   
classifierrE  rF   s     r   r=   z$DinatForImageClassification.__init__  s     $
+ ++'
 FLEVEVYZEZBIIdjj--v/@/@A`b`k`k`m 	
 	r    rJ   labelsr   r   r"  rK   c                 T   ||n| j                   j                  }| j                  ||||      }|d   }| j                  |      }d}	|| j	                  ||| j                         }	|s|f|dd z   }
|	|	f|
z   S |
S t        |	||j                  |j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NrQ  r   rU   )r7   r8   r&   r'   r(   )	rH   rR  r.  r\  loss_functionr6   r&   r'   r(   )rG   rJ   r]  r   r   r"  r   rW  r8   r7   r   s              r   rN   z#DinatForImageClassification.forward  s     &1%<k$++B]B]**/!5#	  
  
/%%ffdkkBDY,F)-)9TGf$EvE)!//))#*#A#A
 	
r    )NNNNN)r)   r*   r+   r=   r   r   r-   r.   
LongTensorr   r   r0   r6   rN   rP   rQ   s   @r   rY  rY    s       59-1,0/3&*)
u001)
 ))*)
 $D>	)

 'tn)
 d^)
 
u00	1)
 )
r    rY  zL
    NAT backbone, to be used with frameworks like DETR and MaskFormer.
    c                   x     e Zd Z fdZd Ze	 	 	 d	dej                  dee	   dee	   dee	   de
f
d       Z xZS )
DinatBackbonec           	      .   t         |   |       t         | 	  |       t        | dg       t	        |      | _        t        |      | _        |j                  gt        t        |j                              D cg c]  }t        |j                  d|z  z         c}z   | _        i }t        | j                  | j                         D ]  \  }}t#        j$                  |      ||<    t#        j&                  |      | _        | j+                          y c c}w )Nr?  rU   )r<   r=   _init_backboner   r:   rM   r  rA  rA   r  r   r  rr   r@  zip_out_featuresr   r   r@   
ModuleDicthidden_states_normsrE  )rG   rH   r  rh  stager]   rI   s         r   r=   zDinatBackbone.__init__  s     v&$
+)&1#F+#--.X]^abhbobo^pXq1rXqST#f6F6FA6M2NXq1rr !#&t'9'94==#IE<)+l)C& $J#%==1D#E  	 2ss   9"Dc                 .    | j                   j                  S r   rH  r   s    r   rI  z"DinatBackbone.get_input_embeddings	  rJ  r    rJ   r   r   r"  rK   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }| j	                  |      }| j                  ||ddd      }|j                  }d}t        | j                  |      D ]  \  }	}
|	| j                  v s|
j                  \  }}}}|
j                  dddd      j                         }
|
j                  |||z  |      }
 | j                  |	   |
      }
|
j                  ||||      }
|
j                  dddd      j                         }
||
fz  } |s|f}|r||j                  fz  }|S t!        ||r|j                  nd|j"                  	      S )
a/  
        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> processor = AutoImageProcessor.from_pretrained("shi-labs/nat-mini-in1k-224")
        >>> model = AutoBackbone.from_pretrained(
        ...     "shi-labs/nat-mini-in1k-224", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)

        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 512, 7, 7]
        ```NT)r   r   r!  r"  r1   r   rU   r   r   )feature_mapsr&   r'   )rH   rR  r   r   rM   rA  r(   re  stage_namesout_featuresrd   re   r   r   rh  r&   r	   r'   )rG   rJ   r   r   r"  rT  r   r&   rl  ri  hidden_stater   r]   rg   rh   r   s                   r   rN   zDinatBackbone.forward  s   B &1%<k$++B]B]$8$D $++JjJj 	 2C1N-TXT_T_TqTq??<8,,/!%59  
  66#&t'7'7#GE<))):F:L:L7
L&%+33Aq!Q?JJL+00Ve^\Z>t77>|L+00VULY+33Aq!Q?JJL/ $H "_F#70022M%3G'//T))
 	
r    )NNN)r)   r*   r+   r=   rI  r   r-   rO   r   r   r	   rN   rP   rQ   s   @r   rb  rb    ss    &0  04,0&*G
llG
 'tnG
 $D>	G

 d^G
 
G
 G
r    rb  )rY  r=  r-  rb  )rx   F)<r,   r   dataclassesr   typingr   r   r-   r   activationsr   modeling_outputsr	   modeling_utilsr
   pytorch_utilsr   r   utilsr   r   r   r   r   r   utils.backbone_utilsr   configuration_dinatr   natten.functionalr   r   
get_loggerr)   loggerr$   r3   r6   rs   r:   r>   rj   rO   r   r   r   r   r   r   r   r   r   r   r  r  r-  r=  rY  rb  __all__r1   r    r   <module>r}     s   @  ! "   ! . - Q  2 , ;;// 
		H	% 
K K K  
K{ K K& 
K K K*bii ,!299 !Hryy 0U\\ e T V[VbVb *%BII %CBII CL
")) 
!")) !H		 	")) 	D DN, ,^C
299 C
L *? * *$ O
% O
 O
d ;
"6 ;
;
| 
_
(- _

_
D ar    