tj&KzddlZddlZddlZddlmZddlmZddl m Z m Z ddl m Z mZmZmZddlmZmZddlmZdd lmZmZmZd d lmZd d lmZmZd d lm Z d dl!m"Z"GddeZ#GddedZ$GddeZ%Gdde Z&GddedZ'Gdde"Z(gdZ)dS)N) functional) BatchFeature)group_images_by_shapereorder_images)ChannelDimensionPILImageResamplingSizeDictget_image_size) ImagesKwargs VideosKwargs) TensorType) VideoMetadatagroup_videos_by_shapereorder_videos) Glm46VConfig)Glm46VImageProcessor smart_resize)Glm46VImageProcessorPil)Glm46VVideoProcessorceZdZdZdZdS) GlmgaConfiga image_start_token_id (`int`, *optional*, defaults to 151339): The image start token index to encode the start of image. image_end_token_id (`int`, *optional*, defaults to 151340): The image end token index to encode the end of image. video_start_token_id (`int`, *optional*, defaults to 151361): The video start token index to encode the start of video. video_end_token_id (`int`, *optional*, defaults to 151362): The video end token index to encode the end of video. ```python >>> from transformers import AutoModelForImageTextToText, GlmgaConfig >>> # Initializing a Glmga style configuration >>> configuration = GlmgaConfig() >>> # Initializing a model (reusing the GLM-4.6V implementation) from that configuration >>> model = AutoModelForImageTextToText.from_config(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```glmgaN)__name__ __module__ __qualname____doc__ model_typee/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/transformers/models/glmga/modular_glmga.pyrr)s  .JJJr!rc<eZdZUdZeed<eed<eed<eed<dS)GlmgaImageProcessorKwargsa patch_size (`int`, *optional*, defaults to 14): The spatial patch size of the vision encoder. temporal_patch_size (`int`, *optional*, defaults to 2): The temporal patch size of the vision encoder. merge_size (`int`, *optional*, defaults to 2): The merge size of the vision encoder to llm encoder. patch_expand_factor (`int`, *optional*, defaults to 1): The patch_expand_factor of the vision encoder to llm encoder. patch_sizetemporal_patch_size merge_sizepatch_expand_factorN)rrrrint__annotations__r r!r"r$r$DsH  OOOOOOr!r$F)totalc eZdZdZdeddededdded ed ed eeezd zd eeezd zdededededed zde e zd zde f dZ d S)GlmgaImageProcessorimagesz torch.Tensor do_resizesizeresample7PILImageResampling | tvF.InterpolationMode | int | None do_rescalerescale_factor do_normalize image_meanN image_stdr%r&r'r(disable_groupingreturn_tensorsreturnc t||\}}i}|D]n\}}|jdd\}}|rPt| ||| | | z| z|j|j\}}||t|||}|||<ot||}t||\}}i}i}|D]O\}}|jdd\}}| |||||| }|j dkr| d}|jd| zd krN|ddd df d| |jd| zz ddd}tj||gd }|jdd \}} }!| | z}"|| z|| z}$}#|||"| |!|#| z| | |$| z| | }|d ddd ddd ddd }|||"|#z|$z|!| z| z| z}%|%||<|"|#|$gg|z||<Qt||}&t||}tj|&d }'tj|}(t'|'|(d|S)z9 Preprocess an image or batch of images. )r9N num_framesheightwidthtemporal_factorfactor min_pixels max_pixelsr@rAr1r2r.rdimrr  pixel_valuesimage_grid_thwdata tensor_type)ritemsshaper shortest_edge longest_edgeresizer rrescale_and_normalizendim unsqueezerepeattorchcatviewpermutereshapetensorr))selfr/r0r1r2r4r5r6r7r8r%r&r'r(r9r:kwargsgrouped_imagesgrouped_images_indexresized_images_groupedrXstacked_imagesr@rAresized_height resized_widthresized_imagesprocessed_images_groupedprocessed_gridspatchesrepeats batch_sizet_lenchannelgrid_tgrid_hgrid_wflatten_patchesprocessed_imagesrRrSs) r" _preprocesszGlmgaImageProcessor._preprocessYsi.0EV^n/o/o/o,,!#%3%9%9%;%; ; ; !E>*05MFE 0<2!$7% 25HH#1#0111- "&"!}MMM%"-"" -; "5 ) )'(>@TUU/D^fv/w/w/w,,#% %3%9%9%;%;) M) M !E>,:,@,E )NM00 NL*V_G|q  !++A..}Q"55::!!!!RSS&/00*gmA.>AT.TUWXZ[]^ )Wg$6A>>>)0rr): &Jw11F+z9=J;VFFll#*$*$  GooaAq!Q1aCCG%oo&(-- :ZGO /> $U +'-vv&>%?*%LOE " ")*BDXYY(:NOOy!1q999 o66"..QQ_m    r!) rrrr(listboolr floatr)strrrr{r r!r"r-r-Vse ^$e e  e L e  e e e DK'$.e 4;&-e e !e e !e +e j(4/!e $ %e e e e e e r!r-ceZdZdZdeejdededddede d ed e ee zd zd e ee zd zd e de de de de e zd zde fdZd S)GlmgaImageProcessorPilr.r/r0r1r2zPILImageResampling | int | Noner4r5r6r7Nr8r(r%r&r'r:r;c g}g}|D]-}|jdd\}}|rPt| ||| | | z| z|j|j\}}||t |||}|r|||}|r|||| }tj |tj }|j dkrtj |d }|j d krtj |d }|jdd\}}|jd | zdkrLtj |ddd df| |jd | zz d }tj||gd }|jdd\}}}|| z}|| z|| z}}|||| ||| z| | || z| | }tj|d }||||z|z|| z| z| z} || d||||g/tj|d }!tj|}"t)|!|"d|S)z? Preprocess images one by one for PIL backend. r=Nr>rFrGdtyperr)axisrHr.rI) rr.rHrLrMrNrrrOrPrQrT)rXrrYrZr[r rescale normalizenpasarrayfloat32r] expand_dimsr_ concatenaterd transposeappendsqueezearrayr)#rfr/r0r1r2r4r5r6r7r8r(r%r&r'r:rgrzrpimager@rArlrm image_arrayrrrsrtrurvrwrxrqryrRrSs# r"r{z"GlmgaImageProcessorPil._preprocesss*D =D =E!K,MFE 0<2!$7% 25HH#1#0111-  !}MMM%$ < UN;; Euj)DD*U"*===K1$$ n[qAAA 1$$ n[qAAA ,7,=bcc,B )NM #&99Q>>)233'';+ &Jw11F+z9=J;VFF!))#*$*$  Gl7,JKKG%oo&(-- :ZGO  # #O$;$;A$>$> ? ? ?  " "FFF#; < < < <~&6Q??? /22"..QQ_m    r!)rrrr(r|rndarrayr}r r~r)rrrr{r r!r"rrsd RZ d d  d 4 d  d d d DK'$.d 4;&-d !d d !d d j(4/d " #d d d d d d r!rc\eZdZUeeefed<eed<eed<eed<eed<eed<dS)GlmgaVideoProcessorInitKwargsmax_image_sizer%r&r'r( max_framesN)rrrdictrr)r*r r!r"rr+sWcN"""OOOOOOOOOOOr!rc (eZdZdddZddiZdZdZdZ dd ed e e zdzfd Z d d de j d d d dddddddfdeejdedededzdddede dede ee zdzde ee zdzde dzde dzde dzde dzdeezdzfdZdS) GlmgaVideoProcessori1i6)rYrZrZrr.iNmetadatafpsc |t|ddtd|jdz }|jpt ||jz dz}||n|j}t ||zt|jd|jz fdtD}krfdtD}nLg}d} d|z } tD]3} || | kr%| | z } | | | || z krn4t|krit|dkrdtdz d} } n|d|d} } tj| | t }nEt|kr2tjddz t }t!g}}|D]0}||vr*||| |1t|dzr| |dtj|S) a Args: metadata (`VideoMetadata`): Metadata of the video containing information about total duration, fps and total number of frames. fps (`int` or `float`, *optional*): Target frames to sample per second. Defaults to `self.fps`. Returns: np.ndarray: Indices to sample video frames. NrzAsked to sample frames per second but no video metadata was provided which is required when sampling in Glmga. Please pass in `VideoMetadata` object or set `do_sample_frames=False`r.cg|]}|zSr r ).0iduration_per_frames r" z5GlmgaVideoProcessor.sample_frames..[sJJJa,,JJJr!cDg|]}tj|zz Sr )mathfloor)r_i extract_t total_framess r"rz5GlmgaVideoProcessor.sample_frames..^s-bbb2TZ\(9I(EFFbbbr!rrIr)getattr ValueErrortotal_num_framesdurationroundrr)minrrangerlenmaxrlinspacetolistsetaddr)rfrrrg max_frame_idxr target_fps timestamps frame_indicescurrent_secondinv_fps frame_indexstartendseenuniqidxrrrs @@@r" sample_framesz!GlmgaVideoProcessor.sample_frames;s  wx==EX   0 $q( $Omhl.J(K(Ka(OOSS :-..  4?33 -JJJJeL6I6IJJJ ) # #bbbbbQVW`QaQabbbMMMN*nG$\22   k*n<<"g-N!((555%G);;; }   ) )=!!Q&&L1$4a 8 8s*1-}R/@sKsISIIIPPRRMM   ) + +KrFrGr) channel_dimr.rIrJrrHrLrMrNrrOrP)pixel_values_videosvideo_grid_thwrT)rrWconvert_to_rgbrXrrYrZrbr[r rr rFIRSTr\r_r`rarcrdrer)/rfrrr0r1r2r4r5r6r7r8r(r%r&r'r:rggrouped_videosgrouped_videos_indexresized_videos_groupedrXstacked_videosBTCHWr?r@rArlrmresized_videosprocessed_videos_groupedrprqrrrsrvrurwrxryprocessed_videosrrrUs/ r"r{zGlmgaVideoProcessor._preprocess~s&0EV/L/L,,!#%3%9%9%;%; ; ; !E> E!%!4!4^!D!D*0MAq!Q()1aJ ]0<)!$7% 25HH#1#0111- "0!4!4QUAq!!D!D!%"!}MMM%"-"" "0!4!4Q1nm!\!\,: "5 ) )'(>@TUU0E^/T/T,,#% %3%9%9%;%;% M% M !E>,:>!;LZjZp,q,q,q )NM"77 NL*V_N%G}Q"55::!!!!RSS&/004G!4KQPQSTUU)Wg$6A>>>*1-*; 'J22F+z9=J;VFFll#*$*$  GooaAq!Q1aCCG%oo&(-- :ZGO /> $U +'-vv&>%?*%LOE " ")*BDXYY(:NOO#i(8a@@@o66#6,   >BBBBr!)N)rrrr1rrr(rrr)r~rr BICUBICr|r`Tensorr}r rrr{r r!r"rr4s&8K L LD$&9:N CJ #'AAA5[4 AAAAL $ $N`Nh )!1504*.!%*.!%26!dCdCU\"dCdC dC o dC L dCdCdCdCDK'$.dC4;&-dC!4ZdC$JdC!4ZdC$JdC j(4/!dCdCdCdCdCdCr!r)rr-rr)*rnumpyrr`torchvision.transforms.v2rtvFimage_processing_utilsrimage_transformsrr image_utilsrr r r processing_utilsr r utilsr video_utilsrrrglm46v.configuration_glm46vrglm46v.image_processing_glm46vrr"glm46v.image_processing_pil_glm46vrglm46v.video_processing_glm46vrrr$r-rrr__all__r r!r"rsz  777777222222EEEEEEEE ;:::::::OOOOOOOOOO666666OOOOOOOOHHHHHHAAAAAA ,6 E$h h h h h .h h h Vg g g g g 4g g g TLnCnCnCnCnC.nCnCnCb   r!