
    <h                     ,   S SK Jr  S SKJrJrJr  S SKrS SKrS SKJ	r	  S SK
Jr  SSKJr  SSKJr  SS	KJrJr  SS
KJr  SSKJrJrJr  SSKJr  SSKJr  SSKJr  \\" SS9 " S S\5      5       5       r " S S\	R@                  5      r!S r"S@S jr#S\RH                  S\%S\RH                  4S jr& SAS\	R@                  S\RH                  S\RH                  S \RH                  S!\\RH                     S"\'S#\'S$\\   4S% jjr( " S& S'\	R@                  5      r) " S( S)\	R@                  5      r* " S* S+\	R@                  5      r+S,\RH                  S-\RH                  S.\RH                  S\RH                  4S/ jr, " S0 S1\	R@                  5      r- " S2 S3\	R@                  5      r.\ " S4 S5\5      5       r/S6\RH                  S7\'S\0\RH                  \RH                  4   4S8 jr1S9\RH                  S:\%S;\%S\RH                  4S< jr2\" S=S9 " S> S?\/5      5       r3S5S?/r4g)B    )	dataclass)CallableOptionalUnionN)nnpad_sequence   )ACT2FN)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstring)can_return_tuple   )AutoModelForKeypointDetection   )LightGlueConfiga  
    Base class for outputs of LightGlue keypoint matching models. Due to the nature of keypoint detection and matching,
    the number of keypoints is not fixed and can vary from image to image, which makes batching non-trivial. In the
    batch of images, the maximum number of matches is set as the dimension of the matches and matching scores. The mask
    tensor is used to indicate which values in the keypoints, matches, matching_scores and prune tensors are keypoint
    matching information.
    )custom_introc                   f   \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S'   Sr\\R                     \	S	'   Sr\\\R                        \	S
'   Sr\\\R                        \	S'   Srg)LightGlueKeypointMatchingOutput&   aW  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*):
    Loss computed during training.
matches (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Index of keypoint matched in the other image.
matching_scores (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Scores of predicted matches.
keypoints (`torch.FloatTensor` of shape `(batch_size, num_keypoints, 2)`):
    Absolute (x, y) coordinates of predicted keypoints in a given image.
prune (`torch.IntTensor` of shape `(batch_size, num_keypoints)`):
    Pruning mask indicating which keypoints are removed and at which layer.
mask (`torch.BoolTensor` of shape `(batch_size, num_keypoints)`):
    Mask indicating which values in matches, matching_scores, keypoints and prune are keypoint matching
    information.
hidden_states (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, 2, num_channels,
    num_keypoints)` returned when `output_hidden_states=True` is passed or when
    `config.output_hidden_states=True`
attentions (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, 2, num_heads, num_keypoints,
    num_keypoints)` returned when `output_attentions=True` is passed or when
    `config.output_attentions=True`
Nlossmatchesmatching_scores	keypointsprunemaskhidden_states
attentions )__name__
__module____qualname____firstlineno____doc__r   r   torchFloatTensor__annotations__r   r   r   r    	IntTensorr!   r"   tupler#   __static_attributes__r$       h/var/www/html/shao/venv/lib/python3.13/site-packages/transformers/models/lightglue/modeling_lightglue.pyr   r   &   s    0 )-D(5$$
%,+/GXe''(/37OXe//07-1Ix))*1'+E8EOO$+(,D(5$$
%,8<M8E%"3"345<59Ju00129r0   r   c                      ^  \ rS rSrS\4U 4S jjr S	S\R                  S\\	   S\
\\R                     \\R                  \R                  4   4   4S jjrSrU =r$ )
LightGluePositionalEncoderS   configc                    > [         TU ]  5         [        R                  " SUR                  UR
                  -  S-  SS9U l        g )Nr   Fbias)super__init__r   Lineardescriptor_dimnum_attention_heads	projectorselfr5   	__class__s     r1   r:   #LightGluePositionalEncoder.__init__T   s:    1f&;&;v?Y?Y&Y]^&^ejkr0   r   output_hidden_statesreturnc                     U R                  U5      nUR                  SSS9n[        R                  " U5      n[        R                  " U5      nXV4nU(       a  XC4nU$ U4nU$ )Nr   dim)r>   repeat_interleaver*   cossin)r@   r   rC   projected_keypoints
embeddingscosinessinesoutputs           r1   forward"LightGluePositionalEncoder.forwardX   sl     #nnY7(::1":E
))J'		*%%
6J*2 R\P]r0   )r>   F)r%   r&   r'   r(   r   r:   r*   Tensorr   boolr   r.   rQ   r/   __classcell__rA   s   @r1   r3   r3   S   sg    l l
 OT		=Ed^		uU\\"E%,,*D$EE	F	 	r0   r3   c                 |    U SS S S24   nU SSS S24   n[         R                  " U* U/SS9R                  S5      nU$ )N.r   r   rF   rG   )r*   stackflatten)xx1x2rot_xs       r1   rotate_halfr`   d   sL    	
3!8B	
319BKK"b	r*2226ELr0   c                 &   U R                   nU R                  5       n UR                  5       nUR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nUR	                  US9UR	                  US94$ )a  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
dtype)rc   float	unsqueezer`   to)	qkrJ   rK   position_idsunsqueeze_dimrc   q_embedk_embeds	            r1   apply_rotary_pos_embrm   l   s    ( GGE		A		A
--
&C
--
&Cw;q>C/0Gw;q>C/0G::E:"GJJUJ$;;;r0   r"   n_reprD   c                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)shapeexpandreshape)r"   rn   batchnum_key_value_headsslenhead_dims         r1   	repeat_kvrw      s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTTr0   modulequerykeyvalueattention_maskscalingdropoutkwargsc                 @   [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub"  US S 2S S 2S S 2S UR
                  S   24   nX-   n
[        R                  R                  U
S[        R                  S9R                  UR                  5      n
[        R                  R                  XU R                  S9n
[        R                  " X5      nUR	                  SS5      R                  5       nX4$ )Nr   r
   rY   rF   )rH   rc   )ptrainingr   )rw   num_key_value_groupsr*   matmul	transposerp   r   
functionalsoftmaxfloat32rf   rc   r~   r   
contiguous)rx   ry   rz   r{   r|   r}   r~   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                r1   eager_attention_forwardr      s     3 ; ;<JU$?$?@L<<';';Aq'ABWLL!$Q1.D
0@0@0D.D%DE#1==((2U]](SVVW\WbWbcL==((6??([L,,|:K''1-88:K$$r0   c                   d  ^  \ rS rSrSrS\S\4U 4S jjr    SS\R                  S\
\\R                  \R                  4      S\
\R                     S	\
\R                     S
\
\R                     S\\   S\\R                  \
\R                     \
\\R                        4   4S jjrSrU =r$ )LightGlueAttention   z=Multi-headed attention from 'Attention Is All You Need' paperr5   	layer_idxc                 P  > [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        g )Nrv   g      Tr7   )r9   r:   r5   r   getattrhidden_sizer=   rv   rt   r   r}   attention_dropout	is_causalr   r;   attention_biasq_projk_projv_projo_projr@   r5   r   rA   s      r1   r:   LightGlueAttention.__init__   sI   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r0   r"   position_embeddingsr|   encoder_hidden_statesencoder_attention_maskr   rD   c                 $   UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	US Ln
U
(       a  UOUnU
(       a  UOUnU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      nUb  Uu  nn[        XUU5      u  p[        nU R                  R                  S:w  a  [        U R                  R                     nU" U U	UUU4U R                  (       d  SOU R                  U R                  S.UD6u  nnUR                  " / UQSP76 R!                  5       nU R#                  U5      nUU4$ )NrF   r   r   eager        )r~   r}   )rp   rv   r   viewr   r   r   rm   r   r5   _attn_implementationr   r   r   r}   rr   r   r   )r@   r"   r   r|   r   r   r   input_shapehidden_shapequery_statesis_cross_attentioncurrent_statescurrent_attention_maskr   r   rJ   rK   attention_interfacer   r   s                       r1   rQ   LightGlueAttention.forward   s    $))#2.88b8$--8{{=166|DNNqRST2$>2D.-;M!7Sa[[055lCMMaQRS
{{>277EOOPQSTU**HC';LVY[^'_$L(?;;++w6"9$++:Z:Z"[$7"	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((r0   )r   r5   rv   r   r   r   r   r   r   r}   r   )NNNN)r%   r&   r'   r(   r)   r   intr:   r*   rT   r   r.   r   r   rQ   r/   rV   rW   s   @r1   r   r      s    G
 
3 
4 LP158<9=*)||*) &eELL%,,,F&GH*) !.	*)
  (5*) !) 6*) -.*) 
u||Xell3XeELL>Q5RR	S*) *)r0   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueMLP   r5   c                 f  > [         TU ]  5         Xl        [        UR                     U l        [        R                  " UR                  UR                  5      U l	        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  SS9U l        g )NT)elementwise_affine)r9   r:   r5   r   
hidden_actactivation_fnr   r;   intermediate_sizefc1r   fc2	LayerNorm
layer_normr?   s     r1   r:   LightGlueMLP.__init__   s{    #F$5$5699V55v7O7OP99V55v7I7IJ,,v'?'?TXYr0   r"   rD   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ N)r   r   r   r   )r@   r"   s     r1   rQ   LightGlueMLP.forward   sB    /6**=9/r0   )r   r5   r   r   r   r%   r&   r'   r(   r   r:   r*   rT   rQ   r/   rV   rW   s   @r1   r   r      s1    Z ZU\\ ell  r0   r   c                     ^  \ rS rSrS\S\4U 4S jjr  SS\R                  S\R                  S\R                  S\	\
   S	\	\
   S
\\R                  \	\\R                        \	\\R                        4   4S jjrSrU =r$ )LightGlueTransformerLayeri  r5   r   c                    > [         TU ]  5         [        X5      U l        [	        U5      U l        [        X5      U l        [	        U5      U l        g r   )r9   r:   r   self_attentionr   self_mlpcross_attention	cross_mlpr   s      r1   r:   "LightGlueTransformerLayer.__init__	  s@    0C$V,1&D%f-r0   descriptorsr   r|   rC   output_attentionsrD   c                    U(       a  SOS nU(       a  SOS nU(       a  Xa4-   nUR                   u  pn
U R                  UUUUS9u  p[        R                  " X/SS9nU R	                  U5      nX-   nU(       a  X4nUR                  SSX5      R                  S5      R                  XU
5      nUb6  UR                  SSSSU	5      R                  S5      R                  USSU	5      OS nU R                  UUUUS9u  nn[        R                  " UU/SS9nU R                  U5      nUU-   nU(       a4  UU4nUUR                  XU
5      4-   W-   UR                  XU
5      4-   U-   nU(       a	  X|4-   U4-   nXU4$ )Nr$   )r   r|   r   rF   rG   r   r   )r   r   r   )	rp   r   r*   catr   rr   flipr   r   )r@   r   r   r|   rC   r   all_hidden_statesall_attentions
batch_sizenum_keypointsr<   attention_outputself_attentionsintermediate_statesoutput_statesself_attention_descriptorsself_attention_hidden_statesr   r   cross_attention_outputcross_attentionscross_intermediate_statescross_output_statescross_attention_hidden_statess                           r1   rQ   !LightGlueTransformerLayer.forward  s    #7BD0d 1N B4?4E4E1
> -1,?,? ))/	 -@ -
) $ii(GRP&9:%0%@",?+O( '..r1mTT!WWZ? 	 ) ""2q!Q>CCAFNNz[\^_ano 	 483G3G&"7#9/	 4H 4
0 0 %*II/IKa.bhj$k!"nn-FG03FF-FH[,\)!-55jQ_`bc./ &&z.QST 0	0  +.@@DTCVVN~==r0   )r   r   r   r   )FF)r%   r&   r'   r(   r   r   r:   r*   rT   r   rU   r.   rQ   r/   rV   rW   s   @r1   r   r     s    . .3 . 05,1H>\\H> <<H> 	H>
 'tnH> $D>H> 
u||XeELL&9:HU5<<EX<YY	ZH> H>r0   r   
similaritymatchability0matchability1c                    U R                   u  p4n[        R                  R                  U5      [        R                  R                  U5      R	                  SS5      -   n[        R                  R                  U S5      n[        R                  R                  U R	                  SS5      R                  5       S5      R	                  SS5      nU R                  X4S-   US-   4S5      n	Xx-   U-   U	SS2SU2SU24'   [        R                  R                  UR                  S5      * 5      U	SS2SS2S4'   [        R                  R                  UR                  S5      * 5      U	SS2SSS24'   U	$ )z;create the log assignment matrix from logits and similarityr   r   rF   rY   r   N)	rp   r   r   
logsigmoidr   log_softmaxr   new_fullsqueeze)
r   r   r   r   num_keypoints_0num_keypoints_1certaintiesscores0scores1scoress
             r1   sigmoid_log_double_softmaxr   [  sM    4>3C3C0J--**=9BMM<T<TUb<c<m<mnoqr<ssKmm''
A6Gmm''
(<(<R(D(O(O(QSTU__`bdfgG  *.A?UVCV!WYZ[F4;4E4SF1 0 00111=3H3H3L2LMF1crc2:11=3H3H3L2LMF1b#2#:Mr0   c                      ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jrS\R                  S\R                  4S jr	S	r
U =r$ )
LightGlueMatchAssignmentLayerij  r5   c                    > [         TU ]  5         UR                  U l        [        R                  " U R                  U R                  SS9U l        [        R                  " U R                  SSS9U l        g )NTr7   r   )r9   r:   r<   r   r;   final_projectionmatchabilityr?   s     r1   r:   &LightGlueMatchAssignmentLayer.__init__k  sY    $33 "		$*=*=t?R?RY] ^IId&9&914Hr0   r   r!   rD   c                    UR                   u  p4nU R                  U5      nU[        R                  " U R                  UR
                  S9S-  -  nUR                  US-  SXE5      nUS S 2S4   nUS S 2S4   nXxR                  SS5      -  n	Ub  UR                  US-  SU5      nUS S 2S4   R                  S5      n
US S 2S4   R                  S5      R                  SS5      nX-  nU	R                  US:H  [        R                  " U	R                  5      R                  5      n	U R                  U5      nUR                  US-  SUS5      nUS S 2S4   nUS S 2S4   n[        XU5      nU$ )Ndeviceg      ?r   r   r   rF   rY   )rp   r   r*   tensorr<   r   rr   r   re   masked_fillfinforc   minr   r   )r@   r   r!   r   r   r<   m_descriptorsm_descriptors0m_descriptors1r   mask0mask1r   matchability_0matchability_1r   s                   r1   rQ   %LightGlueMatchAssignmentLayer.forwardr  sy   4?4E4E1
>--k:%T5H5HQ^QeQe(fjn(nn%--jAoq-`&q!t,&q!t,#&>&>r2&FF
<<
aMBDAJ((,EAJ((,66r2>E=D#//	5;;zGWGW;X;\;\]J ((5#++J!OQqQ%ad+%ad+ ,JWr0   c                     U R                  U5      n[        R                  R                  U5      R	                  S5      nU$ )z0Get matchability of descriptors as a probabilityrF   )r   r   r   sigmoidr   )r@   r   r   s      r1   get_matchability.LightGlueMatchAssignmentLayer.get_matchability  s7    ((5}},,\:BB2Fr0   )r<   r   r   )r%   r&   r'   r(   r   r:   r*   rT   rQ   r  r/   rV   rW   s   @r1   r   r   j  sW    I I5<< u||  4ELL U\\  r0   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueTokenConfidenceLayeri  r5   c                 n   > [         TU ]  5         [        R                  " UR                  S5      U l        g Nr   )r9   r:   r   r;   r<   tokenr?   s     r1   r:   &LightGlueTokenConfidenceLayer.__init__  s&    YYv44a8
r0   r   rD   c                     U R                  UR                  5       5      n[        R                  R	                  U5      R                  S5      nU$ )NrF   )r  detachr   r   r  r   )r@   r   r  s      r1   rQ   %LightGlueTokenConfidenceLayer.forward  s=    

;--/0%%e,44R8r0   )r  r   rW   s   @r1   r  r    s/    9 9
5<< ELL  r0   r  c                   8    \ rS rSr% Sr\\S'   SrSrSr	Sr
SrSrg	)
LightGluePreTrainedModeli  zz
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
r5   	lightgluepixel_valuesFTr$   N)r%   r&   r'   r(   r)   r   r,   base_model_prefixmain_input_namesupports_gradient_checkpointing_supports_flash_attn_supports_sdpar/   r$   r0   r1   r  r    s+    
 #$O&+#Nr0   r  r   	thresholdc                 (   U R                   u  n  nU SS2SS2SS24   R                  S5      nU SS2SS2SS24   R                  S5      nUR                  nUR                  n[        R                  " UR                   S   UR
                  S9S   n[        R                  " UR                   S   UR
                  S9S   n	XR                  SU5      :H  n
XR                  SU5      :H  nUR                  R                  5       nUR                  S5      n[        R                  " XU5      n[        R                  " XR                  SU5      U5      nXU:  -  nXR                  SU5      -  n[        R                  " XS5      n[        R                  " UUS5      n[        R                  " Xg/5      R                  SS5      R                  US-  S5      n[        R                  " X/5      R                  SS5      R                  US-  S5      nUU4$ )z1obtain matches from a score matrix [Bx M+1 x N+1]NrF   r   r   r   r   )rp   maxindicesr*   aranger   gathervaluesexp
new_tensorwhererZ   r   rr   )r   r  r   _max0max1matches0matches1indices0indices1mutual0mutual1zeromatching_scores0matching_scores1valid0valid1r   r   s                      r1   get_matches_from_scoresr1    s   ||J1!SbS#2#+""1%D!SbS#2#+""1%D||H||H ||HNN1-hooFtLH||HNN1-hooFtLH//!X66G//!X66G ;;??D??1D{{7$7{{7,C,CAx,PRVW945F}}Q11F {{6R0H{{68R0Hkk8./99!Q?GG
UVXZ[Gkk#3"FGQQRSUVW__`jmn`nprsOO##r0   r   heightwidthc                     [         R                  " X!/U R                  U R                  S9S   nUS-  nUR	                  S5      R
                  S-  nXSSSS24   -
  US   -  n U $ )a}  
Normalize keypoints locations based on image image_shape

Args:
    keypoints (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`):
        Keypoints locations in (x, y) format.
    height (`int`):
        Image height.
    width (`int`):
        Image width.

Returns:
    Normalized keypoints locations of shape (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`).
r   rc   Nr   rF   .).NN)r*   r   r   rc   r  r  )r   r2  r3  sizeshiftscales         r1   normalize_keypointsr9    sl     <<	0@0@	XY]^D1HEHHRL!#E3a<00E/4JJIr0   zV
    LightGlue model taking images as inputs and outputting the matching of them.
    c                   >  ^  \ rS rSrSrS\4U 4S jjrS\S\4S jr	 S$S\
R                  S	\
R                  S
\\   S\\
R                  \\
R                  \
R                  4   4   4S jjrS\
R                  S\S\
R                  S\
R                  S\
R                  4
S jrS%S jrS\
R                  S\
R                  S\S\
R                  4S jrS\
R                  S	\
R                  S\
R                  S\
R                  S\
R                  S\
R                  S\4S jrS rS\
R                  S\
R                  S\
R                  S\
R                  S\\
R                  \
R                  4   4
S jr   S&S	\
R                  S\
R                  S\S\S\
R                  S\\   S
\\   S\\
R                  \
R                  \
R                  \\4   4S jjr\\   S&S \
R2                  S!\\
R4                     S\\   S
\\   S\\\4   4
S" jj5       5       rS#rU =r$ )'LightGlueForKeypointMatchingi  aF  
LightGlue is a model matching keypoints in images by leveraging detections from a keypoint detector such as
SuperPoint. It is based on the SuperGlue architecture and is designed to be lightweight and efficient.
It consists of :
    1. Keypoint Encoder
    2. A Graph Neural Network with self and cross attention layers
    3. Matching Assignment layers

The correspondence ids use -1 to indicate non-matching points.

Philipp Lindenberger, Paul-Edouard Sarlin and Marc Pollefeys. LightGlue: Local Feature Matching at Light Speed.
In ICCV 2023. https://arxiv.org/pdf/2306.13643.pdf
r5   c           
      2  > [         TU ]  U5        [        R                  " UR                  UR
                  S9U l        UR                  R                  U l        UR                  U l	        UR                  U l        UR                  U l        UR                  U l        UR                  U l        U R                  U R                  :w  a0  [        R                   " U R                  U R                  SS9U l        O[        R$                  " 5       U l        ['        U5      U l        [        R*                  " [-        UR                  5       Vs/ sH  n[/        XS9PM     sn5      U l        [        R*                  " [-        UR                  5       Vs/ sH  n[3        U5      PM     sn5      U l        [        R*                  " [-        UR                  S-
  5       Vs/ sH  n[7        U5      PM     sn5      U l        U R;                  5         g s  snf s  snf s  snf )N)trust_remote_codeTr7   )r   r   )r9   r:   r   from_configkeypoint_detector_configr=  keypoint_detectordescriptor_decoder_dim keypoint_detector_descriptor_dimr<   num_hidden_layers
num_layersfilter_thresholddepth_confidencewidth_confidencer   r;   input_projectionIdentityr3   positional_encoder
ModuleListranger   transformer_layersr   match_assignment_layersr  token_confidence	post_init)r@   r5   ir#  rA   s       r1   r:   %LightGlueForKeypointMatching.__init__  s    !>!J!J++v?W?W"
 170O0O0f0f-$33 22 & 7 7 & 7 7 & 7 7$"G"GG$&IId.S.SUYUhUhos$tD!$&KKMD!"<V"D"$--EJ6KcKcEdeEd&v;Ede#
 (*}}<A&BZBZ<[\<[q*62<[\(
$ !#<A&BZBZ]^B^<_`<_q*62<_`!
 	 f ] as   
H
HHlayer_indexrD   c                     SS[         R                  " SU-  U R                  -  5      -  -   n[         R                  " USS5      $ )z-scaled confidence threshold for a given layerg?g?g      r   r   )npr   rD  clip)r@   rS  r  s      r1   _get_confidence_threshold6LightGlueForKeypointMatching._get_confidence_threshold  s;    #tk'9DOO'K LLL	wwy!Q''r0   r   r   rC   c                     UR                  5       R                  5       nU R                  U5      nU R                  X#S9nXE4$ )NrC   )r  r   rH  rJ  )r@   r   r   rC   projected_descriptorskeypoint_encoding_outputs         r1   _keypoint_processing1LightGlueForKeypointMatching._keypoint_processing  sI     "((*557 $ 5 5k B#'#:#:9#:#p $>>r0   keypoint_confidencesr!   
num_pointsc                 l   UR                   u  pVX R                  S-
  :  ap  UR                  US:H  S5      nUR                  US-  S5      nU R	                  U5      nSX:  R                  5       R                  SS9U-  -
  nXR                  :  n	U	$ [        R                  " U[        R                  S9n	U	$ )zRevaluate whether we should stop inference based on the confidence of the keypointsr   r   r   rF   g      ?rG   rb   )rp   rD  r   rr   rW  rd   sumrF  r*   onesrU   )
r@   r_  rS  r!   r`  r   r#  r  ratio_confidentearly_stopped_pairss
             r1   _get_early_stopped_image_pairs;LightGlueForKeypointMatching._get_early_stopped_image_pairs%  s     


1,, $8#C#CDAIq#Q #7#?#?
aQS#T 66{CI!%9%E$L$L$N$R$RWX$R$Y\f$ffO"14I4I"I
 #" #(**Zuzz"J""r0   c                 v    Ub  X   nX$   nU R                   U   " X5      n[        XPR                  5      u  pgXg4$ r   )rN  r1  rE  )r@   r   r!   rS  early_stopsr   r   r   s           r1   _get_keypoint_matching3LightGlueForKeypointMatching._get_keypoint_matching8  sI    "%2K$D--k:;M#:6CXCX#Y ''r0   confidencesr   c                 \    USU R                   -
  :  nUb  XAU R                  U5      :*  -  nU$ )z#mask points which should be removedr   )rG  rW  )r@   rl  r   rS  keeps        r1   _get_pruning_mask.LightGlueForKeypointMatching._get_pruning_mask@  s:    T2223"4#A#A+#NNNDr0   r  prune_outputc                   ^ UR                   u  n  n	U R                  U   R                  U5      n
U R                  XjU5      mTR	                  US:H  [
        R                  " S5      5      mU4S jXS   US   TU4 5       u  ppn[        U5       H  nUUUU   4==   S-  ss'   M     S XX4 5       u  ppX4n[        USSS9nUUXU4$ )	zv
For a given layer, prune keypoints based on the confidence of the keypoints and the matchability of the
descriptors.
r   Fc              3   r   >#    U H'  n[        UT5       VVs/ sH	  u  p#X#   PM     snnv   M)     g s  snnf 7fr   )zip).0r   tr!   pruned_keypoints_masks       r1   	<genexpr>JLightGlueForKeypointMatching._do_layer_keypoint_pruning.<locals>.<genexpr>[  s<      c
c %(0E$FG$FQW$FGc Hs   717r   c              3   4   #    U H  n[        US S9v   M     g7f)T)batch_firstNr   )ru  pruned_tensors     r1   rx  ry  c  s      S
!j D9!js   TrF   r{  padding_value)	rp   rN  r  ro  r   r*   r   rL  r	   )r@   r   r   r!   r  rq  r_  rS  r   r#  descriptors_matchabilitypruned_descriptorspruned_keypoints_0pruned_keypoints_1pruned_maskpruned_indicesrQ  pruned_keypointsrw  s                     @r1   _do_layer_keypoint_pruning7LightGlueForKeypointMatching._do_layer_keypoint_pruningG  s    ',,
Aq#'#?#?#L#]#]^i#j  $ 6 67Kgr s 5 A A$!)U\\Z_M` ac
&!ilDY[bcc
_0BQ_ z"AN1--.!3. #S
"4J\!jS
O0B /C%n$VXY!#3^R^^^r0   c                    ^ [         R                  " T5      mS XB4 5       u  pBS XS4 5       u  pSU4S jUUUU4 5       u  pEp#X#XE4$ )Nc              3   6   #    U H  n[        US SS9v   M     g7f)TrF   r}  Nr   ru  r   s     r1   rx  MLightGlueForKeypointMatching._concat_early_stopped_outputs.<locals>.<genexpr>u  s       3
C TDC   c              3   6   #    U H  n[        US SS9v   M     g7f)Tr   r}  Nr   r  s     r1   rx  r  y  s       >
N TCNr  c              3   ,   >#    U H
  nUT   v   M     g 7fr   r$   )ru  r   early_stops_indicess     r1   rx  r  }  s#      g
 &'s   )r*   rZ   )r@   r  final_pruned_keypoints_indices!final_pruned_keypoints_iterationsr   r   s    `    r1   _concat_early_stopped_outputs:LightGlueForKeypointMatching._concat_early_stopped_outputsl  st     $kk*=>3
"C3
/>
*N>
:g
 .1	g
c"@ .RYjjr0   r   r   r   c                   ^ UR                   u  mnU4S jXU4 5       u  pnUS S 2S4   nUS S 2S4   nUS S 2S4   nUS S 2S4   n	US S 2S4   n
US S 2S4   n[        R                  " TS-  SU4SUR                  UR                  S9n[        R
                  " TS-  SU4UR                  UR                  S9n[        TS-  5       H  n[        R                  " X   S:H  SX~   R                  SX   R                  SS95      5      XSXn   4'   [        R                  " X   S:H  SXn   R                  SX   R                  SS95      5      XSX~   4'   X   XSXn   4'   X   XSX~   4'   M     X4$ )Nc              3   L   >#    U H  oR                  TS -  S S5      v   M     g7f)r   rF   N)rr   )ru  r   r   s     r1   rx  JLightGlueForKeypointMatching._do_final_keypoint_pruning.<locals>.<genexpr>  s'      -
AdvNN:?Ar22Ads   !$r   r   r   rF   r5  )r   )
rp   r*   fullr   rc   zerosrL  r"  r  clamp)r@   r  r   r   r   r#  r(  r)  r&  r'  r-  r.  _matches_matching_scoresrQ  r   s                  @r1   _do_final_keypoint_pruning7LightGlueForKeypointMatching._do_final_keypoint_pruning  s     
A-
BITcAd-
)/ 1a4=1a4=1a4=1a4=*1a40*1a40 ::zQ=A2gnndkdqdqr ;;1_a/oNcNc
 zQ'A*/++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' +0++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' 3C2E8;./2B2E8;./ ( ))r0   r2  r3  r   c           
      p	  ^( U(       a  SOS nU(       a  SOS n	UR                   S   S:X  aQ  UR                   S S n
UR                  U
S[        R                  S9UR	                  U
5      UR	                  U
5      UU	4$ UR
                  nUR                   u  pp[        R                  " UR                  US5      SS9nUR                  US-  US5      nUb  UR                  US-  U5      OS nUR                  US-  XR                  5      n[        R                  " US-  US9n[        XU5      nU R                  X!US	9u  nnUS   nU R                  S:  nU R                  S:  n/ n/ n/ n/ n/ n[        R                  " SXS9R                  US-  S5      n[        R                  " U5      n[!        U R"                  5       GHR  nUR%                  5       nUb  U R'                  UU5      nO$[        R(                  " UUS
   4UR
                  S9nU R*                  U   " UUUUUS9nUu  nnn U(       a  UU-   nU(       a  U	U -   n	U(       Ga  UU R"                  S-
  :  a'  U R,                  U   " U5      n!U R/                  U!UX_S9n"O#[        R(                  " U[        R0                  S9n"[        R2                  " U"5      (       Ga  U"R5                  S5      m(UT(   n#U R7                  X%UT(S9u  n$n%UR9                  [;        U#5      5        UR9                  [;        U$5      5        UR9                  [;        U%5      5        U(       a:  UR9                  [;        UT(   5      5        UR9                  [;        UT(   5      5        UU")    n[=        U(4S jX!S   US   UU4 5       5      u  nn&n'nnU&U'4nU(       a  [=        U(4S jUUW!4 5       5      u  nnn![        R>                  " U"5      (       a    O+U(       d  GM6  U RA                  UUUUUW!U5      u  p!nnnGMU     U(       a9  U(       a2  U RC                  UUUUU5      u  nnnnU RE                  UUUU5      u  nnOEU R7                  X%U R"                  S-
  5      u  nn[        R                  " U5      U R"                  -  nUR                  USU5      nUUUUU	4$ )Nr$   r   r   rF   rb   r   rG   r   rZ  rY   )r|   rC   r   )r`  )ri  c              3   .   >#    U H  nUT)    v   M     g 7fr   r$   ru  r   ri  s     r1   rx  ALightGlueForKeypointMatching._match_image_pair.<locals>.<genexpr>  s       V&dF |,&d   c              3   .   >#    U H  nUT)    v   M     g 7fr   r$   r  s     r1   rx  r    s$      l+ #K<0+r  )#rp   r   r*   r   	new_zerosr   rb  rr   rB  r  r9  r]  rF  rG  rq   	ones_likerL  rD  r6  get_extended_attention_maskrc  rM  rO  rf  rU   anyrI   rj  extendlistr.   allr  r  r  ))r@   r   r   r2  r3  r!   r   rC   r   r   rp   r   r   r#  initial_num_keypointsnum_points_per_pairimage_indicesr\  do_early_stopdo_keypoint_pruningr  r   r   r  r  pruned_keypoints_indicespruned_keypoints_iterationsrS  r   extended_attention_masklayer_outputr"   	attentionr_  re  early_stopped_image_indicesearly_stopped_matchesearly_stopped_matching_scoreskeypoints_0
keypoint_1ri  s)                                           @r1   _match_image_pair.LightGlueForKeypointMatching._match_image_pair  sK    #7BD0d??1"OOCR(E""5"EII">##E*##E*!  !!2;///
,#iiZ(D!L%%j1n6KQO	FJFVt||JN,AB\`!))*q.:OQvQvwZ!^FC'	5A	040I0I9M 1J 1
-- -Q/	 --1 #33a7 )+&,.)#(<<3H#X#_#_`jmn`npr#s &+oo6N&O# 1K%**,K*.*J*J4Q\*]'*/**j+b/5R[d[k[k*l'22;?6%9"3L 5A1K	#$5$E! !/)!;1!44+/+@+@+Mk+Z( +/*M*M,k4 +N +'
 +0**Zuzz*R'99011 #6"G"G"JK2?2L/KOKfKf#;K Lg LH)+H (..t4O/PQNN4(=#>?#**40M+NO*6==dC[\gCh>ij9@@FabmFnAop +>?R>R*S'PU V'2aL)A,PTVc&dV QMKj$ "-j 9I*fk l !9 ; 4+l gc02MOc 99011"" 33#!03,# d(@$HcQ 2h 0 22'25# h*,MwXg (,'F'F.%	($G_ (,'B'B;VZVeVehiVi'j$G_050PSWSbSb0b-,M,U,U0-
)
 -
 	
r0   r  labelsc                 $   S nUb  [        S5      eUb  UOU R                  R                  nUb  UOU R                  R                  nUR                  S:w  d  UR                  S5      S:w  a  [        S5      eUR                  u  pgpn
UR                  US-  XU
5      nU R                  U5      nUS S u  ppUR                  USSS5      R                  U5      nUR                  USSU R                  5      R                  U5      nUR                  USS5      nUR                  5       nUS S 2S S 2S S 2S4   U
-  US S 2S S 2S S 2S4'   US S 2S S 2S S 2S4   U	-  US S 2S S 2S S 2S4'   U R                  UUU	U
UUUS	9u  nnnnn[        UUUUUUUUS
9$ )Nz9LightGlue is not trainable, no labels should be provided.   r   r   zOInput must be a 5D tensor of shape (batch_size, 2, num_channels, height, width)   rF   r   )r!   r   rC   )r   r   r   r   r    r!   r"   r#   )
ValueErrorr5   r   rC   ndimr6  rp   rr   r@  rf   rB  cloner  r   )r@   r  r  r   rC   r   r   r#  channelsr2  r3  keypoint_detectionsr   r   r!   absolute_keypointsr   r   r    r"   r#   s                        r1   rQ   $LightGlueForKeypointMatching.forwardZ  s    XYY1B1N-TXT_T_TqTq$8$D $++JjJj 	 !\%6%6q%9Q%>noo1=1C1C.
x#++JNHeT"44\B*=bq*A'	k%%j!R;>>|L	!))*aT=b=bcffgst||J2.&__.);Aq!QJ)G%)O1aA:&);Aq!QJ)G&)P1aA:&EIE[E[/!5 F\ F
B%
 /+'!	
 		
r0   )rF  r<   rE  rH  r@  rB  rN  rD  rJ  rO  rM  rG  rS   r   )NNN) r%   r&   r'   r(   r)   r   r:   r   rd   rW  r*   rT   r   rU   r.   r]  rf  rj  ro  r  r  r  r  r   r   r+   
LongTensorr   r   rQ   r/   rV   rW   s   @r1   r;  r;    s    @(S (U ( jo? <<?49LL?X`aeXf?	u||U5<<#=>>	??#$)LL#?B#JO,,#didpdp#	#&(U\\ 5<< ^a fkfrfr #_\\#_ <<#_ ll	#_
 #_ ll#_ $ll#_ #_Jk8#*#* #* 	#*
 ||#* 
u||U\\)	*#*V ",0/3k
<<k
 \\k
 	k

 k
 llk
 $D>k
 'tnk
 
u||U\\5<<E	Fk
Z  .2,0/33
''3
 ))*3
 $D>	3

 'tn3
 
u55	63
  3
r0   r;  r
  )r   )5dataclassesr   typingr   r   r   numpyrU  r*   r   torch.nn.utils.rnnr	   activationsr   modeling_flash_attention_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr   auto.modeling_autor   configuration_lightgluer   r   Moduler3   r`   rm   rT   r   rw   rd   r   r   r   r   r   r   r  r  r.   r1  r9  r;  __all__r$   r0   r1   <module>r     sU  ( " , ,    + ! B F & D D - > 4  :k  :  :F "<<	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 U\\*% % % '(%4D) D)N299 "P>		 P>f-2\\JO,,
\\&BII &R	BII 	   $ELL $U $uU\\[`[g[gMgGh $@5<<  S U\\ , 
f
#; f

f
R &'E
Fr0   