§
    }Štjz  ã                   óÂ  — U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m	Z	 d dl
mZ d dlmZ d dlmZmZ d dlmZ ddlmZmZmZ dd	lmZmZmZmZ dd
lmZmZ ddlmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& erddl'm(Z(  ej)        e*¦  «        Z+dZ,dZ-g d¢Z.e/e0         e1d<   dZ2dZ3dZ4dZ5dZ6dZ7de/e         ddfd„Z8ddddddddœddd e9d!e9ez  d"e9d#e9dz  d$e:dz  d%e/e9         e9z  dz  d&e/e9         e9z  dz  d'e0dz  d(e:d)e0fd*„Z; G d+„ d,e j<        ¦  «        Z=e>eef         Z? G d-„ d.¦  «        Z@d/e@ddd e9d"e9d#e9f
d0„ZAd/e@de>e=e/e?         f         dz  fd1„ZBd2e?ddfd3„ZCd4e/e?         ddd e9d"e9d#e9ddfd5„ZDd4e/e?         ddd e9d"e9d#e9ddfd6„ZEd4e/e?         ddd e9d"e9d#e9ddfd7„ZF G d8„ d9e¦  «        ZGd2e?deGfd:„ZHd;d<de/e?         fd=„ZId;d<d>e0de/e?         fd?„ZJd@e9ddfdA„ZKdS )Bé    N)Údatetime)ÚPath)ÚLock)ÚTYPE_CHECKINGÚAny)Úquoteé   )ÚCommitOperationAddÚ
UploadInfoÚ_fetch_upload_modes)ÚLocalUploadFileMetadataÚLocalUploadFilePathsÚget_local_upload_pathsÚread_upload_metadata)ÚDEFAULT_REVISIONÚ
REPO_TYPES)ÚDEFAULT_IGNORE_PATTERNSÚ_format_sizeÚfilter_repo_objectsÚtqdm)Úis_xet_available)Úsha_fileobj)ÚHfApié
   éd   )
é   é2   éK   r   é}   éÈ   éú   i�  iX  iè  ÚCOMMIT_SIZE_SCALEé   i † i'  r    r   Ú
paths_listÚreturnc                 óè  — t                                d¦  «         t          | ¦  «        t          k    r5t                                dt          | ¦  «        d›dt          d›d�¦  «         ddlm}  |d„ ¦  «        }| D ]Ä}t          |j        ¦  «        }|j	        }t          |j        ¦  «        d	k    rt          |j        ¦  «        nd	}||         d
xx         dz  cc<   t          |dd…         ¦  «        D ]K\  }}|dk    rd	nd                     |d|…         ¦  «        }||         d                              |¦  «         ŒLŒÅ|                     ¦   «         D ]o\  }	}
|
d
         }t          |
d         ¦  «        }||z   }|t           k    r=|	d	k    rdn|	}t                                d|› d|d›d|d›d|d›dt           d›d�¦  «         Œpg }g }| D ]t}|j                             ¦   «         j        }|dz  }|t(          k    r|                     |j        |f¦  «         ŒM|t,          k    r|                     |j        |f¦  «         Œu|rŠd                     d„ |dd…         D ¦   «         ¦  «        }t          |¦  «        dk    rdt          |¦  «        dz
  › d�nd}t                                dt          |¦  «        › dt(          › d |› |› d!�¦  «         |rŠd                     d"„ |dd…         D ¦   «         ¦  «        }t          |¦  «        dk    rdt          |¦  «        dz
  › d�nd}t                                dt          |¦  «        › d#t,          › d$|› |› d%�¦  «         t                                d&¦  «         dS )'as  
    Validate upload against repository limits and warn about potential issues.

    Args:
        paths_list: List of file paths to be uploaded

    Warns about:
        - Too many files in the repository (>100k)
        - Too many entries (files or subdirectories) in a single folder (>10k)
        - Files exceeding size limits (>20GB recommended, >200GB maximum)
    z/Running validation checks on files to upload...zYou are about to upload ú,z. files. This exceeds the recommended limit of zÍ files per repository.
Consider:
  - Splitting your data into multiple repositories
  - Using fewer, larger files (e.g., parquet files)
  - See: https://huggingface.co/docs/hub/repositories-recommendationsr   )Údefaultdictc                  ó$   — dt          ¦   «         dœS )Nr   )ÚfilesÚsubdirs)Úset© ó    úb/var/www/html/CA-Chatbot/venv/lib/python3.11/site-packages/huggingface_hub/_upload_large_folder.pyú<lambda>z)_validate_upload_limits.<locals>.<lambda>U   s   € ÀqÕUXÑUZÔUZÐ=[Ð=[€ r.   ú.r*   r	   Néÿÿÿÿú/r+   ÚrootzFolder 'z' contains z
 entries (z files and z/ subdirectories). This exceeds the recommended z< entries per folder.
Consider reorganising into sub-folders.i Êš;z
  - c              3   ó.   K  — | ]\  }}|› d |d›d�V — ŒdS ©z: z.1fÚGBNr-   ©Ú.0ÚpathÚsizes      r/   ú	<genexpr>z*_validate_upload_limits.<locals>.<genexpr>ƒ   s:   è è € Ð!aÐ!a¹j¸dÀD TÐ"9Ð"9¨TÐ"9Ð"9Ð"9Ð"9Ð!aÐ!aÐ!aÐ!aÐ!aÐ!ar.   é   z
  ... and z more filesÚ úFound z files exceeding the zGB recommended maximum:
  - z4
Consider splitting these files into smaller chunks.c              3   ó.   K  — | ]\  }}|› d |d›d�V — ŒdS r6   r-   r8   s      r/   r<   z*_validate_upload_limits.<locals>.<genexpr>�   s:   è è € Ð!\Ð!\¹j¸dÀD TÐ"9Ð"9¨TÐ"9Ð"9Ð"9Ð"9Ð!\Ð!\Ð!\Ð!\Ð!\Ð!\r.   z files larger than zGB (recommended limit):
  - z2
Large files may slow down loading and processing.zValidation checks complete.)ÚloggerÚinfoÚlenÚMAX_FILES_PER_REPOÚwarningÚcollectionsr(   r   Úpath_in_repoÚpartsÚstrÚparentÚ	enumerateÚjoinÚaddÚitemsÚMAX_FILES_PER_FOLDERÚ	file_pathÚstatÚst_sizeÚMAX_FILE_SIZE_GBÚappendÚRECOMMENDED_FILE_SIZE_GB)r$   r(   Úentries_per_folderÚpathsr:   rH   rJ   ÚiÚchildÚfolderÚdataÚ
file_countÚsubdir_countÚtotal_entriesÚfolder_displayÚlarge_filesÚvery_large_filesr;   Úsize_gbÚ	files_strÚmore_strs                        r/   Ú_validate_upload_limitsre   8   s“  € õ ‡K‚KÐAÑBÔBÐBõ ˆ:�„Õ+Ò+Ð+Ý�ŠðU¥s¨:¡¤ÐBð Uð UÝ5GÐcðUð Uð Uñ	
ô 	
ð 	
ð (Ð'Ð'Ð'Ð'Ð'à)4¨Ð5[Ð5[Ñ)\Ô)\Ðàð =ð =ˆÝ�EÔ&Ñ'Ô'ˆØ”
ˆõ &)¨¬Ñ%5Ô%5¸Ò%<Ð%<•�T”[Ñ!Ô!Ð!À#ˆØ˜6Ô" 7Ð+Ð+Ô+¨qÑ0Ð+Ð+Ñ+õ " %¨¨¨¤*Ñ-Ô-ð 	=ð 	=‰HˆAˆuØ šF˜F�S�S¨¯ª°°r¸°r´Ñ(;Ô(;ˆFØ˜vÔ& yÔ1×5Ò5°eÑ<Ô<Ð<Ð<ð	=ð
 +×0Ò0Ñ2Ô2ð ð ‰ˆ�Ø˜'”]ˆ
Ý˜4 	œ?Ñ+Ô+ˆØ" \Ñ1ˆàÕ/Ò/Ð/Ø'-°¢} }˜V˜V¸&ˆNÝ�NŠNð:˜>ð :ð :°mÐPð :ð :ØÐOð:ð :Ø-9ÐOð:ð :å0DÐ^ð:ð :ð :ñô ð øð €KØÐàð >ð >ˆØŒ×#Ò#Ñ%Ô%Ô-ˆØ˜Ñ&ˆàÕ%Ò%Ð%Ø×#Ò# UÔ%7¸Ð$AÑBÔBÐBÐBØÕ/Ò/Ð/Ø×Ò Ô 2°GÐ<Ñ=Ô=Ð=øð ð 
Ø—M’MÐ!aÐ!aÐL\Ð]_Ð^_Ð]_ÔL`Ð!aÑ!aÔ!aÑaÔaˆ	ÝLOÐP`ÑLaÔLaÐdeÒLeÐLeÐH¥#Ð&6Ñ"7Ô"7¸!Ñ";ÐHÐHÐHÐHÐkmˆÝ�ŠðC•SÐ)Ñ*Ô*ð Cð CÕAQð Cð CØðCØ&ðCð Cð Cñ	
ô 	
ð 	
ð ð 
Ø—M’MÐ!\Ð!\ÈKÐXZÐYZÐXZÌOÐ!\Ñ!\Ô!\Ñ\Ô\ˆ	ÝGJÈ;ÑGWÔGWÐZ[ÒG[ÐG[ÐC¥# kÑ"2Ô"2°QÑ"6ÐCÐCÐCÐCÐacˆÝ�ŠðA•S˜Ñ%Ô%ð Að AÕ:Rð Að AØðAØ&ðAð Að Añ	
ô 	
ð 	
õ ‡K‚KÐ-Ñ.Ô.Ð.Ð.Ð.r.   Té<   )ÚrevisionÚprivateÚallow_patternsÚignore_patternsÚnum_workersÚprint_reportÚprint_report_everyÚapir   Úrepo_idÚfolder_pathÚ	repo_typerg   rh   ri   rj   rk   rl   rm   c                óò  ‡ ‡‡‡‡‡— ‰€t          d¦  «        ‚‰t          vrt          dt          › �¦  «        ‚‰€t          Št          ‰¦  «                             ¦   «                              ¦   «         Š‰                     ¦   «         st          d‰› d�¦  «        ‚|€g }nt          |t          ¦  «        r|g}|t          z  }|€(t          j        ¦   «         pd}t          |dz  d¦  «        }‰                      ‰‰|d¬	¦  «        }t                               d
|› �¦  «         |j        Š	 ‰                      ‰‰‰¬¦  «        }t%          |¦  «        }|dk    r*t                               dd› d|› d‰› d‰› dd› d�¦  «         n4# t(          $ r'}t                               d|› �¦  «         Y d}~nd}~ww xY wt-          ¦   «         rt.          nt0          }t3          ˆfd„‰                     d¦  «        D ¦   «         ||¬¦  «        }ˆfd„|D ¦   «         }t                               dt%          |¦  «        › d�¦  «         t7          |¦  «         t                               d¦  «         ˆfd„t9          |d¬¦  «        D ¦   «         }t;          ||¦  «        Šˆ ˆˆˆˆfd„t=          |¦  «        D ¦   «         }|D ]}|                     ¦   «          Œ|	r$tA          d‰ !                    ¦   «         z   ¦  «         tE          j"        ¦   «         }	 tE          j#        d¦  «         tE          j"        ¦   «         |z
  |
k    r6|	r!tI          ‰ !                    ¦   «         ¦  «         tE          j"        ¦   «         }‰ %                    ¦   «         rt                               d ¦  «         nŒ”|D ]}| &                    ¦   «          Œt                               ‰ !                    ¦   «         ¦  «         t                               d!¦  «         dS )"zŒUpload a large folder to the Hub in the most resilient way possible.

    See [`HfApi.upload_large_folder`] for the full documentation.
    Nz For large uploads, `repo_type` is explicitly required. Please set it to `model`, `dataset` or `space`. If you are using the CLI, pass it as `--repo-type=model`.z"Invalid repo type, must be one of zProvided path: 'z' is not a directoryr	   é   T)ro   rq   rh   Úexist_okzRepo created: )ro   rq   rg   iô  ú
zP================================================================================z
WARNING: This repository has aR   commits.
Repositories with a large number of commits can experience performance issues.

Consider squashing your commit history using `super_squash_history()`.
To do so, you need to stop this process, run the snippet below and restart the upload command.  from huggingface_hub import super_squash_history
  super_squash_history(repo_id='z', repo_type='zÎ')

Note: This is a non-revertible operation. See the documentation for more details:
https://huggingface.co/docs/huggingface_hub/main/en/package_reference/hf_api#huggingface_hub.HfApi.super_squash_history
zCould not check commit count: c              3   óŽ   •K  — | ]?}|                      ¦   «         ¯|                     ‰¦  «                             ¦   «         V — Œ@d S ©N)Úis_fileÚrelative_toÚas_posix)r9   r:   rp   s     €r/   r<   z/upload_large_folder_internal.<locals>.<genexpr>ä   sO   øè è € ÐiÐi°dÐZ^×ZfÒZfÑZhÔZhÐiˆ×	Ò	˜+Ñ	&Ô	&×	/Ò	/Ñ	1Ô	1ÐiÐiÐiÐiÐiÐir.   z**/*)ri   rj   c                 ó0   •— g | ]}t          ‰|¦  «        ‘ŒS r-   )r   )r9   Úrelpathrp   s     €r/   ú
<listcomp>z0upload_large_folder_internal.<locals>.<listcomp>è   s$   ø€ ÐbÐbÐbÀ7Õ(¨°gÑ>Ô>ÐbÐbÐbr.   r?   z candidate files to uploadzStarting upload...c                 ó>   •— g | ]}|t          ‰|j        ¦  «        f‘ŒS r-   )r   rG   )r9   rW   rp   s     €r/   r}   z0upload_large_folder_internal.<locals>.<listcomp>ñ   s<   ø€ ð ð ð àð 
Õ$ [°%Ô2DÑEÔEÐFðð ð r.   zRecovering from metadata files)Údescc                 óR   •— g | ]#}t          j        t          ‰‰‰‰‰d œ¬¦  «        ‘Œ$S ))Ústatusrn   ro   rq   rg   )ÚtargetÚkwargs)Ú	threadingÚThreadÚ_worker_job)r9   Ú_rn   ro   rq   rg   r�   s     €€€€€r/   r}   z0upload_large_folder_internal.<locals>.<listcomp>ø   sY   ø€ ð ð ð ð õ 	ÔÝà ØØ"Ø&Ø$ðð ð		
ñ 		
ô 		
ðð ð r.   z

zIs done: exiting main loopzUpload is complete!)'Ú
ValueErrorr   r   r   Ú
expanduserÚresolveÚis_dirÚ
isinstancerI   r   ÚosÚ	cpu_countÚmaxÚcreate_reporA   rB   ro   Úlist_repo_commitsrC   rE   Ú	ExceptionÚdebugr   ÚUPLOAD_BATCH_SIZE_XETÚUPLOAD_BATCH_SIZE_LFSr   Úglobre   r   ÚLargeUploadStatusÚrangeÚstartÚprintÚcurrent_reportÚtimeÚsleepÚ_print_overwriteÚis_donerL   )rn   ro   rp   rq   rg   rh   ri   rj   rk   rl   rm   Únb_coresÚrepo_urlÚcommitsÚcommit_countÚeÚupload_batch_sizeÚfiltered_paths_listr$   rN   ÚthreadsÚthreadÚlast_report_tsr�   s   `````                  @r/   Úupload_large_folder_internalrª   ˜   s¬  øøøøøø€ ð& ÐÝðIñ
ô 
ð 	
ð �
Ð"Ð"ÝÐJ½jÐJÐJÑKÔKÐKØÐÝ#ˆå�{Ñ#Ô#×.Ò.Ñ0Ô0×8Ò8Ñ:Ô:€KØ×ÒÑÔð OÝÐM¨KÐMÐMÐMÑNÔNÐNàÐØˆˆÝ	�O¥SÑ	)Ô	)ð ,Ø*Ð+ˆØÕ.Ñ.€OàÐÝ”<‘>”>Ð& QˆÝ˜( a™-¨Ñ+Ô+ˆð �Š w¸)ÈWÐ_cˆÑdÔd€HÝ
‡K‚KÐ+ Ð+Ð+Ñ,Ô,Ð,ØÔ€Gð;Ø×'Ò'°À9ÐW_Ð'Ñ`Ô`ˆÝ˜7‘|”|ˆØ˜#ÒÐÝ�NŠNð �Xð  ð  Ø0<ð ð  ð 4;ð ð  ð KTð ð  ð ð ð  ð  ñô ð øøõ ð ;ð ;ð ;å�ŠÐ9°aÐ9Ð9Ñ:Ô:Ð:Ð:Ð:Ð:Ð:Ð:øøøøð;øøøõ
 2BÑ1CÔ1CÐ^Õ-Ð-ÕI^Ðõ .ØiÐiÐiÐi¸k×>NÒ>NÈvÑ>VÔ>VÐiÑiÔiØ%Ø'ðñ ô Ðð
 cÐbÐbÐbÐNaÐbÑbÔb€JÝ
‡K‚KÐD�˜Z™œÐDÐDÐDÑEÔEÐEõ ˜JÑ'Ô'Ð'å
‡K‚KÐ$Ñ%Ô%Ð%ðð ð ð å˜*Ð+KÐLÑLÔLðñ ô €Eõ ˜uÐ&7Ñ8Ô8€Fðð ð ð ð ð ð ð õ �{Ñ#Ô#ðñ ô €Gð ð ð ˆØ�Š‰Œˆˆð ð 0Ýˆf�v×,Ò,Ñ.Ô.Ñ.Ñ/Ô/Ð/Ý”Y‘[”[€NðÝŒ
�1‰ŒˆÝŒ9‰;Œ;˜Ñ'Ð+=Ò=Ð=Øð :Ý  ×!6Ò!6Ñ!8Ô!8Ñ9Ô9Ð9Ý!œY™[œ[ˆNØ�>Š>ÑÔð 	Ý�KŠKÐ4Ñ5Ô5Ð5Øðð ð ð ˆØ�Š‰Œˆˆå
‡K‚K�×%Ò%Ñ'Ô'Ñ(Ô(Ð(Ý
‡K‚KÐ%Ñ&Ô&Ð&Ð&Ð&s   Ä*AF Æ
F3ÆF.Æ.F3c                   ó¤   — e Zd Z ej        ¦   «         Z ej        ¦   «         Z ej        ¦   «         Z ej        ¦   «         Z ej        ¦   «         Z	dS )Ú	WorkerJobN)
Ú__name__Ú
__module__Ú__qualname__ÚenumÚautoÚSHA256ÚGET_UPLOAD_MODEÚPREUPLOAD_LFSÚCOMMITÚWAITr-   r.   r/   r¬   r¬   #  sO   € € € € € ØˆTŒY‰[Œ[€FØ�d”i‘k”k€OØ�D”I‘K”K€MØˆTŒY‰[Œ[€FØˆ4Œ9‰;Œ;€D€D€Dr.   r¬   c                   ól   — e Zd ZdZddee         defd„Zdefd„Zde	d	ed
e
ddfd„Zdefd„Zde	fd„ZdS )r—   zBContains information, queues and tasks for a large upload process.r	   rN   r¥   c                 óJ  — || _         t          j        ¦   «         | _        t          j        ¦   «         | _        t          j        ¦   «         | _        t          j        ¦   «         | _        t          ¦   «         | _        d| _	        d| _
        d| _        || _        d| _        d| _        d | _        t!          j        ¦   «         | _        d| _        t          ¦   «         | _        | j         D ]½}|\  }}|j        €| j                             |¦  «         Œ)|j        €| j                             |¦  «         ŒK|j        dk    r"|j        s| j                             |¦  «         Œx|j        s| j                             |¦  «         Œšt4                               d|j        › d�¦  «         Œ¾d S )Nr   r	   ÚlfszSkipping file z! (already uploaded and committed))rN   ÚqueueÚQueueÚqueue_sha256Úqueue_get_upload_modeÚqueue_preupload_lfsÚqueue_commitr   ÚlockÚnb_workers_sha256Únb_workers_get_upload_modeÚnb_workers_preupload_lfsr¥   Únb_workers_commitÚnb_workers_waitingÚlast_commit_attemptr   ÚnowÚ_started_atÚ
_chunk_idxÚ_chunk_lockÚsha256ÚputÚupload_modeÚis_uploadedÚis_committedrA   r“   rG   )ÚselfrN   r¥   ÚitemrW   Úmetadatas         r/   Ú__init__zLargeUploadStatus.__init__1  sŽ  € ØˆŒ
Ý7<´{±}´}ˆÔÝ@EÄÁÄˆÔ"Ý>C¼k¹m¼mˆÔ Ý7<´{±}´}ˆÔÝ‘F”FˆŒ	à&'ˆÔØ/0ˆÔ'Ø-.ˆÔ%Ø&7ˆÔØ&'ˆÔØ'(ˆÔØ15ˆÔ å#œ<™>œ>ˆÔØ ˆŒÝ!%¡¤ˆÔð ”Jð 	eð 	eˆDØ"‰OˆE�8ØŒÐ&ØÔ!×%Ò% dÑ+Ô+Ð+Ð+ØÔ%Ð-ØÔ*×.Ò.¨tÑ4Ô4Ð4Ð4ØÔ%¨Ò.Ð.°xÔ7KÐ.ØÔ(×,Ò,¨TÑ2Ô2Ð2Ð2ØÔ*ð eØÔ!×%Ò% dÑ+Ô+Ð+Ð+å—’Ðc¨eÔ.@ÐcÐcÐcÑdÔdÐdÐdð	eð 	er.   r%   c                 óh   — | j         5  t          | j                 cd d d ¦  «         S # 1 swxY w Y   d S rw   )rÊ   r"   rÉ   ©rÐ   s    r/   Útarget_chunkzLargeUploadStatus.target_chunkS  sy   € ØÔð 	6ð 	6Ý$ T¤_Ô5ð	6ð 	6ð 	6ð 	6ñ 	6ô 	6ð 	6ð 	6ð 	6ð 	6ð 	6ð 	6øøøð 	6ð 	6ð 	6ð 	6ð 	6ð 	6s   ˆ'§+®+ÚsuccessÚnb_itemsÚdurationNc           
      ó¶  — | j         5  |s/t                               d|› d�¦  «         | xj        dz  c_        nJ|t          | j                 k    r4|dk     r.t                               d|› d�¦  «         | xj        dz  c_        t          dt          | j        t          t          ¦  «        dz
  ¦  «        ¦  «        | _        d d d ¦  «         d S # 1 swxY w Y   d S )NzFailed to commit z9 files at once. Will retry with less files in next batch.r	   é(   zSuccessfully committed z. at once. Increasing the limit for next batch.r   )	rÊ   rA   rE   rÉ   r"   rB   r�   ÚminrC   )rÐ   r×   rØ   rÙ   s       r/   Úupdate_chunkzLargeUploadStatus.update_chunkW  s0  € ØÔð 	Wð 	WØð %Ý—’Ðv°8ÐvÐvÐvÑwÔwÐwØ�” 1Ñ$�”�ØÕ.¨t¬Ô?Ò?Ð?ÀHÈrÂMÀMÝ—’Ðn°hÐnÐnÐnÑoÔoÐoØ�” 1Ñ$�”å! !¥S¨¬½#Õ>OÑ:PÔ:PÐSTÑ:TÑ%UÔ%UÑVÔVˆDŒOð	Wð 	Wð 	Wñ 	Wô 	Wð 	Wð 	Wð 	Wð 	Wð 	Wð 	Wð 	Wøøøð 	Wð 	Wð 	Wð 	Wð 	Wð 	Ws   ˆB9CÃCÃCc                 ó  — d}d}d}d}d}d}d}d}d}	d}
d}| j         5  | j        D ]\  }}|j        r|
dz  }
Œ|	|j        z  }	|dz  }|j        �|dz  }||j        z  }|j        dk    r|dz  }|j        €|dz  }|j        r|dz  }||j        z  }|j        r|dz  }||j        z  }Œ€t          |	¦  «        }t          j
        ¦   «         }|                     d¦  «        }|| j        z
  }t          |¦  «                             d¦  «        d         }d}|d|› d	|› d
�z  }|dz  }|dz  }|d|› d|› d	t          |¦  «        › d|› d�	z  }|d|› d|› d	t          |¦  «        › d|› d�	z  }|dk    r	|d|› d�z  }|d|› d|› d	t          |¦  «        › d|› d�	z  }|d|
› d�z  }|dz  }|d| j        › d�z  }|d| j        › d�z  }|d| j        › d�z  }|d| j        › d�z  }|d| j        › d�z  }|dz  }|cddd¦  «         S # 1 swxY w Y   dS )z<Generate a report of the current status of the large upload.r   r	   Nr¹   z%Y-%m-%d %H:%M:%Sr1   z
----------ú z (z) z----------
z	Files:   zhashed r3   z) | zpre-uploaded: ú)z (+z unsure)z | committed: z | ignored: ru   z	Workers: z	hashing: z | zget upload mode: zpre-uploading: zcommitting: z	waiting: z3---------------------------------------------------)rÀ   rN   Úshould_ignorer;   rË   rÍ   rÎ   rÏ   r   r   rÇ   ÚstrftimerÈ   rI   ÚsplitrÁ   rÂ   rÃ   rÄ   rÅ   )rÐ   Ú	nb_hashedÚsize_hashedÚnb_preuploadedÚnb_lfsÚnb_lfs_unsureÚsize_preuploadedÚnb_committedÚsize_committedÚ
total_sizeÚignored_filesÚtotal_filesr‡   rÒ   Útotal_size_strrÇ   Únow_strÚelapsedÚelapsed_strÚmessages                       r/   r›   z LargeUploadStatus.current_reportb  ss  € àˆ	ØˆØˆØˆØˆØÐØˆØˆØˆ
ØˆØˆàŒYð /	ð /	Ø#œzð 4ð 4‘��8ØÔ)ð Ø! QÑ&�MØØ˜hœmÑ+�
Ø˜qÑ �Ø”?Ð.Ø ‘N�IØ 8¤=Ñ0�KØÔ'¨5Ò0Ð0Ø˜a‘K�FØÔ'Ð/Ø! QÑ&�MØÔ'ð 6Ø" aÑ'�NØ$¨¬Ñ5Ð$ØÔ(ð 4Ø  AÑ%�LØ" h¤mÑ3�NøÝ)¨*Ñ5Ô5ˆNå”,‘.”.ˆCØ—l’lÐ#6Ñ7Ô7ˆGØ˜DÔ,Ñ,ˆGÝ˜g™,œ,×,Ò,¨SÑ1Ô1°!Ô4ˆKà%ˆGØÐ5˜7Ð5Ð5 kÐ5Ð5Ð5Ñ5ˆGØ�Ñ&ˆGà�{Ñ"ˆGØÐl ÐlÐl¨[ÐlÐl½LÈÑ<UÔ<UÐlÐlÐXfÐlÐlÐlÑlˆGØÐu¨ÐuÐu¸ÐuÐuÅ<ÐP`ÑCaÔCaÐuÐuÐdrÐuÐuÐuÑuˆGØ˜qÒ Ð ØÐ8 Ð8Ð8Ð8Ñ8�ØÐv¨ÐvÐv°{ÐvÐvÅlÐSaÑFbÔFbÐvÐvÐesÐvÐvÐvÑvˆGØÐ7 mÐ7Ð7Ð7Ñ7ˆGà�{Ñ"ˆGØÐ> 4Ô#9Ð>Ð>Ð>Ñ>ˆGØÐO¨4Ô+JÐOÐOÐOÑOˆGØÐK¨Ô)FÐKÐKÐKÑKˆGØÐA dÔ&<ÐAÐAÐAÑAˆGØÐ> 4Ô#:Ð>Ð>Ð>Ñ>ˆGØ�xÑˆGàð_/	ð /	ð /	ð /	ñ /	ô /	ð /	ð /	ð /	ð /	ð /	ð /	øøøð /	ð /	ð /	ð /	ð /	ð /	s   žGG:Ç:G>ÈG>c                 ó€   — | j         5  t          d„ | j        D ¦   «         ¦  «        cd d d ¦  «         S # 1 swxY w Y   d S )Nc              3   ó8   K  — | ]\  }}|j         p|j        V — Œd S rw   ©rÏ   rá   ©r9   r‡   rÒ   s      r/   r<   z,LargeUploadStatus.is_done.<locals>.<genexpr>£  s1   è è € ÐeÐeÉ;È1Èh�xÔ,ÐF°Ô0FÐeÐeÐeÐeÐeÐer.   )rÀ   ÚallrN   rÕ   s    r/   rŸ   zLargeUploadStatus.is_done¡  s    € ØŒYð 	fð 	fÝÐeÐeÐZ^ÔZdÐeÑeÔeÑeÔeð	fð 	fð 	fð 	fñ 	fô 	fð 	fð 	fð 	fð 	fð 	fð 	føøøð 	fð 	fð 	fð 	fð 	fð 	fs   ˆ3³7º7)r	   )r­   r®   r¯   Ú__doc__ÚlistÚ
JOB_ITEM_TÚintrÓ   rÖ   ÚboolÚfloatrÝ   rI   r›   rŸ   r-   r.   r/   r—   r—   .  sß   € € € € € ØLÐLð eð  e˜d :Ô.ð  eÀ3ð  eð  eð  eð  eðD6˜cð 6ð 6ð 6ð 6ð	W Dð 	W°Cð 	WÀ5ð 	WÈTð 	Wð 	Wð 	Wð 	Wð= ð =ð =ð =ð =ð~f˜ð fð fð fð fð fð fr.   r—   r�   c                 ó	  — 	 d}t          | ¦  «        }|€dS |\  }}|xt          j        k    rÐ |d         }	 t          |¦  «         | j                             |¦  «         nk# t          $ r ‚ t          $ rT}	t           	                    d|	› �¦  «         t          j        ¦   «          | j                             |¦  «         Y d}	~	nd}	~	ww xY w| j        5  | xj        dz  c_        ddd¦  «         n# 1 swxY w Y   �nŽxt          j        k    �r 	 t!          |||||¬¦  «         nQ# t          $ r ‚ t          $ r:}	t           	                    d|	› �¦  «         t          j        ¦   «          Y d}	~	nd}	~	ww xY w|D ]r}|\  }
}|j        rŒ|j        xdk    r | j                             |¦  «         Œ7d	k    r| j                             |¦  «         ŒW	 | j                             |¦  «         Œs| j        5  | xj        dz  c_        ddd¦  «         n# 1 swxY w Y   �noxt          j        k    r× 	 t/          |||||¬¦  «         |D ]}| j                             |¦  «         Œnp# t          $ r ‚ t          $ rY}	t           	                    d
|	› �¦  «         t          j        ¦   «          |D ]}| j                             |¦  «         ŒY d}	~	nd}	~	ww xY w| j        5  | xj        dz  c_        ddd¦  «         n# 1 swxY w Y   �nˆxt          j        k    �r  t5          j        ¦   «         }d}	 t7          |||||¬¦  «         nr# t          $ r ‚ t          $ r[}	t           	                    d|	› �¦  «         t          j        ¦   «          |D ]}| j                             |¦  «         Œd}Y d}	~	nd}	~	ww xY wt5          j        ¦   «         |z
  }|                      |t;          |¦  «        |¦  «         | j        5  t5          j        ¦   «         | _        | xj        dz  c_        ddd¦  «         n# 1 swxY w Y   nWt          j         k    rHt5          j!        tD          ¦  «         | j        5  | xj#        dz  c_#        ddd¦  «         n# 1 swxY w Y   �Œ‹)aª  
    Main process for a worker. The worker will perform tasks based on the priority list until all files are uploaded
    and committed. If no tasks are available, the worker will wait for 10 seconds before checking again.

    If a task fails for any reason, the item(s) are put back in the queue for another worker to pick up.

    Read `upload_large_folder` docstring for more information on how tasks are prioritized.
    TNr   zFailed to compute sha256: r	   )rn   ro   rq   rg   zFailed to get upload mode: r¹   ÚregularzFailed to preupload LFS: zFailed to commit: F)$Ú_determine_next_jobr¬   r²   Ú_compute_sha256r½   rÌ   ÚKeyboardInterruptr’   rA   ÚerrorÚ	tracebackÚ
format_excr¼   rÀ   rÁ   r³   Ú_get_upload_moderá   rÍ   r¾   r¿   rÂ   r´   Ú_preupload_lfsrÃ   rµ   rœ   Ú_commitrÝ   rC   rÆ   rÄ   r¶   r�   ÚWAITING_TIME_IF_NO_TASKSrÅ   )r�   rn   ro   rq   rg   Únext_jobÚjobrN   rÑ   r¤   r‡   rÒ   Ústart_tsr×   rÙ   s                  r/   r†   r†   ¦  s¯  € ð]3Ø>Bˆõ ' vÑ.Ô.ˆØÐØˆFØ‰
ˆˆUð Ø!•Ô!Ò!Ð!Ð!Ø˜Q”x�ð2Ý# DÑ)Ô)Ð)ØÔ0×4Ò4°TÑ:Ô:Ð:Ð:øÝ(ð ð ð ØÝ ð 2ð 2ð 2Ý—L’LÐ!A¸aÐ!AÐ!AÑBÔBÐBÝÔ(Ñ*Ô*Ð*ØÔ'×+Ò+¨DÑ1Ô1Ð1Ð1Ð1Ð1Ð1Ð1øøøøð2øøøð
 ”[ð 2ð 2ØÐ,Ô,°Ñ1Ð,Ô,ð2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2ùð +•Ô*Ò*Ñ*Ð*ð+Ý$ U°¸WÐPYÐdlÐmÑmÔmÐmÐmøÝ(ð ð ð ØÝ ð +ð +ð +Ý—L’LÐ!B¸qÐ!BÐ!BÑCÔCÐCÝÔ(Ñ*Ô*Ð*Ð*Ð*Ð*Ð*Ð*øøøøð+øøøð "ð 
Cð 
C�DØ"&‘K�A�xØÔ-ð !Ø Ø"Ô.Ø"˜UšU˜U˜UØ"Ô6×:Ò:¸4Ñ@Ô@Ð@Ð@Ø&šY˜YØ"Ô/×3Ò3°DÑ9Ô9Ð9Ð9ØØ"Ô8×<Ò<¸TÑBÔBÐBÐBà”[ð ;ð ;ØÐ5Ô5¸Ñ:Ð5Ô5ð;ð ;ð ;ñ ;ô ;ð ;ð ;ð ;ð ;ð ;ð ;øøøð ;ð ;ð ;ð ;ùð )•Ô(Ò(Ð(Ð(ð
=Ý" 5¨c¸7ÈiÐbjÐkÑkÔkÐkØ %ð 6ð 6˜ØÔ+×/Ò/°Ñ5Ô5Ð5Ð5ð6øå(ð ð ð ØÝ ð =ð =ð =Ý—L’LÐ!@¸QÐ!@Ð!@ÑAÔAÐAÝÔ(Ñ*Ô*Ð*Ø %ð =ð =˜ØÔ2×6Ò6°tÑ<Ô<Ð<Ð<ð=ð =ð =ð =ð =øøøøð=øøøð ”[ð 9ð 9ØÐ3Ô3°qÑ8Ð3Ô3ð9ð 9ð 9ñ 9ô 9ð 9ð 9ð 9ð 9ð 9ð 9øøøð 9ð 9ð 9ð 9ùð "•Ô!Ò!Ñ!Ð!Ýœ9™;œ;�Ø�ð	$Ý˜E s°GÀyÐ[cÐdÑdÔdÐdÐdøÝ(ð ð ð ØÝ ð $ð $ð $Ý—L’LÐ!9°aÐ!9Ð!9Ñ:Ô:Ð:ÝÔ(Ñ*Ô*Ð*Ø %ð 6ð 6˜ØÔ+×/Ò/°Ñ5Ô5Ð5Ð5Ø#�G�G�G�G�G�Gøøøøð$øøøõ  œ9™;œ;¨Ñ1�Ø×#Ò# G­S°©Z¬Z¸ÑBÔBÐBØ”[ð 2ð 2Ý15´±´�FÔ.ØÐ,Ô,°Ñ1Ð,Ô,ð2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2øõ ”’�Ý”
Õ3Ñ4Ô4Ð4Ø”[ð 3ð 3ØÐ-Ô-°Ñ2Ð-Ô-ð3ð 3ð 3ñ 3ô 3ð 3ð 3ð 3ð 3ð 3ð 3øøøð 3ð 3ð 3ð 3ñy]3s¼   ·)A! Á!C	Á5A
CÃC	ÃC0Ã0C4Ã7C4ÄD% Ä%E3Ä90E.Å.E3Ç2HÈHÈHÈ.3I" É"KÉ6AK
Ë
KËK6Ë6K:Ë=K:Ì+M  Í N/ÍAN*Î*N/Ï3)P(Ð(P,Ð/P,Ñ#R Ò RÒRc                 óL  — | j         5  | j        dk    r¬| j                             ¦   «         dk    r�| j        �ˆt          j        ¦   «         | j        z
  dk    ri| xj        dz  c_        t                               d¦  «         t          j	        t          | j        |                      ¦   «         ¦  «        fcd d d ¦  «         S | j        dk    r†| j                             ¦   «         dk    ri| xj        dz  c_        t                               d¦  «         t          j	        t          | j        |                      ¦   «         ¦  «        fcd d d ¦  «         S | j                             ¦   «         t          k    re| xj        dz  c_        t                               dt          › d�¦  «         t          j        t          | j        t          ¦  «        fcd d d ¦  «         S | j                             ¦   «         | j        k    rg| j        dk    r\| xj        dz  c_        t                               d	¦  «         t          j        t          | j        | j        ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    ra| j        dk    rV| xj        dz  c_        t                               d
¦  «         t          j        t/          | j        ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    rg| j        dk    r\| xj        dz  c_        t                               d¦  «         t          j        t          | j        t          ¦  «        fcd d d ¦  «         S | j                             ¦   «         | j        k    r\| xj        dz  c_        t                               d¦  «         t          j        t          | j        | j        ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    rV| xj        dz  c_        t                               d¦  «         t          j        t/          | j        ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    r\| xj        dz  c_        t                               d¦  «         t          j        t          | j        t          ¦  «        fcd d d ¦  «         S | j                             ¦   «         dk    r\| xj        dz  c_        t                               d¦  «         t          j        t          | j        | j        ¦  «        fcd d d ¦  «         S | j        dk    r¬| j                             ¦   «         dk    r�| j        �ˆt          j        ¦   «         | j        z
  dk    ri| xj        dz  c_        t                               d¦  «         t          j	        t          | j        |                      ¦   «         ¦  «        fcd d d ¦  «         S | j        dk    rþ| j                             ¦   «         dk    rá| j                             ¦   «         dk    rÄ| j                             ¦   «         dk    r§| j                             ¦   «         dk    rŠ| j        dk    r| j        dk    rt| j        dk    ri| xj        dz  c_        t                               d¦  «         t          j	        t          | j        |                      ¦   «         ¦  «        fcd d d ¦  «         S t1          d„ | j        D ¦   «         ¦  «        r(t                               d¦  «         	 d d d ¦  «         d S | xj        dz  c_        t                               dt8          › d�¦  «         t          j        g fcd d d ¦  «         S # 1 swxY w Y   d S )Nr   i,  r	   z;Job: commit (more than 5 minutes since last commit attempt)é–   zJob: commit (>100 files ready)zJob: get upload mode (>z files ready)z5Job: preupload LFS (no other worker preuploading LFS)z.Job: sha256 (no other worker computing sha256)z:Job: get upload mode (no other worker getting upload mode)zJob: preupload LFSzJob: sha256zJob: get upload moderf   z-Job: commit (1 min since last commit attempt)zJob: commitc              3   ó8   K  — | ]\  }}|j         p|j        V — Œd S rw   rö   r÷   s      r/   r<   z&_determine_next_job.<locals>.<genexpr>t  s1   è è € ÐaÐaÁ[ÀQÈ�Ô&Ð@¨(Ô*@ÐaÐaÐaÐaÐaÐar.   z.All files have been processed! Exiting worker.zNo task available, waiting... (zs))rÀ   rÄ   r¿   ÚqsizerÆ   rœ   rA   r“   r¬   rµ   Ú_get_nrÖ   r½   ÚMAX_NB_FILES_FETCH_UPLOAD_MODErÂ   r³   r¾   r¥   rÃ   r´   r¼   rÁ   r²   Ú_get_onerø   rN   rB   rÅ   r
  r¶   )r�   s    r/   r  r    s,	  € Ø	Œð f(ð f(ð Ô$¨Ò)Ð)ØÔ#×)Ò)Ñ+Ô+¨aÒ/Ð/ØÔ*Ð6Ý”	‘”˜fÔ8Ñ8¸6ÒAÐAàÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠLÐVÑWÔWÐWÝÔ$¥f¨VÔ-@À&×BUÒBUÑBWÔBWÑ&XÔ&XÐYðf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð Ô%¨Ò*Ð*¨vÔ/B×/HÒ/HÑ/JÔ/JÈcÒ/QÐ/QØÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠLÐ9Ñ:Ô:Ð:ÝÔ$¥f¨VÔ-@À&×BUÒBUÑBWÔBWÑ&XÔ&XÐYð!f(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð& Ô)×/Ò/Ñ1Ô1Õ5SÒSÐSØÐ-Ô-°Ñ2Ð-Ô-Ý�LŠLÐ`Õ3QÐ`Ð`Ð`ÑaÔaÐaÝÔ-­v°fÔ6RÕTrÑ/sÔ/sÐtð-f(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð2 Ô'×-Ò-Ñ/Ô/°6Ô3KÒKÐKÐPVÔPoÐstÒPtÐPtØÐ+Ô+¨qÑ0Ð+Ô+Ý�LŠLÐPÑQÔQÐQÝÔ+­V°FÔ4NÐPVÔPhÑ-iÔ-iÐjð9f(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð> Ô ×&Ò&Ñ(Ô(¨1Ò,Ð,°Ô1IÈQÒ1NÐ1NØÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠLÐIÑJÔJÐJÝÔ$¥h¨vÔ/BÑ&CÔ&CÐDðEf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðJ Ô)×/Ò/Ñ1Ô1°AÒ5Ð5¸&Ô:[Ð_`Ò:`Ð:`ØÐ-Ô-°Ñ2Ð-Ô-Ý�LŠLÐUÑVÔVÐVÝÔ-­v°fÔ6RÕTrÑ/sÔ/sÐtðQf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðV Ô'×-Ò-Ñ/Ô/°6Ô3KÒKÐKØÐ+Ô+¨qÑ0Ð+Ô+Ý�LŠLÐ-Ñ.Ô.Ð.ÝÔ+­V°FÔ4NÐPVÔPhÑ-iÔ-iÐjð]f(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðb Ô ×&Ò&Ñ(Ô(¨1Ò,Ð,ØÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠL˜Ñ'Ô'Ð'ÝÔ$¥h¨vÔ/BÑ&CÔ&CÐDðif(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðn Ô)×/Ò/Ñ1Ô1°AÒ5Ð5ØÐ-Ô-°Ñ2Ð-Ô-Ý�LŠLÐ/Ñ0Ô0Ð0ÝÔ-­v°fÔ6RÕTrÑ/sÔ/sÐtðuf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðz Ô'×-Ò-Ñ/Ô/°!Ò3Ð3ØÐ+Ô+¨qÑ0Ð+Ô+Ý�LŠLÐ-Ñ.Ô.Ð.ÝÔ+­V°FÔ4NÐPVÔPhÑ-iÔ-iÐjðAf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ðH Ô$¨Ò)Ð)ØÔ#×)Ò)Ñ+Ô+¨aÒ/Ð/ØÔ*Ð6Ý”	‘”˜fÔ8Ñ8¸6ÒAÐAàÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠLÐHÑIÔIÐIÝÔ$¥f¨VÔ-@À&×BUÒBUÑBWÔBWÑ&XÔ&XÐYðWf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð` Ô$¨Ò)Ð)ØÔ#×)Ò)Ñ+Ô+¨aÒ/Ð/ØÔ#×)Ò)Ñ+Ô+¨qÒ0Ð0ØÔ,×2Ò2Ñ4Ô4¸Ò9Ð9ØÔ*×0Ò0Ñ2Ô2°aÒ7Ð7ØÔ(¨AÒ-Ð-ØÔ1°QÒ6Ð6ØÔ/°1Ò4Ð4àÐ$Ô$¨Ñ)Ð$Ô$Ý�LŠL˜Ñ'Ô'Ð'ÝÔ$¥f¨VÔ-@À&×BUÒBUÑBWÔBWÑ&XÔ&XÐYðwf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(õ| ÐaÐaÐTZÔT`ÐaÑaÔaÑaÔað 	(Ý�KŠKÐHÑIÔIÐIØðAf(ð f(ð f(ñ f(ô f(ð f(ð f(ð f(ðH Ð%Ô%¨Ñ*Ð%Ô%Ý�LŠLÐWÕ;SÐWÐWÐWÑXÔXÐXÝ”N BÐ'ðMf(ð f(ð f(ð f(ñ f(ô f(ð f(ð f(ð f(ð f(ð f(ð f(øøøð f(ð f(ð f(ð f(ð f(ð f(sl   ˆB+^Ã B^ÅA:^ÇA<^É!A1^ËA7^Í#A1^Ï!A&^ÑA,^ÓA,^ÕB*^×=C<^Ü9^ÝA ^Þ^Þ ^rÑ   c                 óø   — | \  }}|j         €X|j                             d¦  «        5 }t          |¦  «                             ¦   «         |_         ddd¦  «         n# 1 swxY w Y   |                     |¦  «         dS )z1Compute sha256 of a file and save it in metadata.NÚrb)rË   rP   Úopenr   ÚhexÚsave)rÑ   rW   rÒ   Úfs       r/   r  r  „  s¬   € à�O€Eˆ8Ø„ÐØŒ_×!Ò! $Ñ'Ô'ð 	3¨1Ý)¨!™nœn×0Ò0Ñ2Ô2ˆHŒOð	3ð 	3ð 	3ñ 	3ô 	3ð 	3ð 	3ð 	3ð 	3ð 	3ð 	3øøøð 	3ð 	3ð 	3ð 	3à‡M‚M�%ÑÔÐÐÐs   §'AÁAÁ!ArN   c           
      ó<  — d„ | D ¦   «         }t          ||||                     ¦   «         t          |d¬¦  «        |j        ¬¦  «         t	          | |¦  «        D ]C\  }}|\  }}	|j        |	_        |j        |	_        |j	        |	_
        |	                     |¦  «         ŒDdS )zmGet upload mode for each file and update metadata.

    Also receive info if the file should be ignored.
    c                 ó,   — g | ]}t          |¦  «        ‘ŒS r-   ©Ú_build_hacky_operation©r9   rÑ   s     r/   r}   z$_get_upload_mode.<locals>.<listcomp>’  ó!   € Ð@Ð@Ð@°$Õ'¨Ñ-Ô-Ð@Ð@Ð@r.   r>   )Úsafe)Ú	additionsrq   ro   Úheadersrg   ÚendpointN)r   Ú_build_hf_headersr   r$  ÚzipÚ_upload_moderÍ   Ú_should_ignorerá   Ú_remote_oidÚ
remote_oidr  )
rN   rn   ro   rq   rg   r"  rÑ   ÚadditionrW   rÒ   s
             r/   r  r  �  sÄ   € ð
 AÐ@¸%Ð@Ñ@Ô@€IÝØØØØ×%Ò%Ñ'Ô'Ý�x bÐ)Ñ)Ô)Ø”ðñ ô ð õ ˜e YÑ/Ô/ð ð ‰ˆˆhØ‰ˆˆxØ'Ô4ˆÔØ!)Ô!8ˆÔØ&Ô2ˆÔØ�Š�eÑÔÐÐðð r.   c                 ó˜   — d„ | D ¦   «         }|                      ||||¬¦  «         | D ]!\  }}d|_        |                     |¦  «         Œ"dS )z(Preupload LFS files and update metadata.c                 ó,   — g | ]}t          |¦  «        ‘ŒS r-   r  r  s     r/   r}   z"_preupload_lfs.<locals>.<listcomp>¥  r   r.   )ro   rq   rg   r"  TN)Úpreupload_lfs_filesrÎ   r  ©rN   rn   ro   rq   rg   r"  rW   rÒ   s           r/   r  r  £  sz   € à@Ð@¸%Ð@Ñ@Ô@€IØ×ÒØØØØð	 ñ ô ð ð !ð ð ‰ˆˆxØ#ˆÔØ�Š�eÑÔÐÐðð r.   c                 óš   — d„ | D ¦   «         }|                      ||||d¬¦  «         | D ]!\  }}d|_        |                     |¦  «         Œ"dS )zCommit files to the repo.c                 ó,   — g | ]}t          |¦  «        ‘ŒS r-   r  r  s     r/   r}   z_commit.<locals>.<listcomp>´  r   r.   z(Add files using upload-large-folder tool)ro   rq   rg   Ú
operationsÚcommit_messageTN)Úcreate_commitrÏ   r  r/  s           r/   r	  r	  ²  s}   € à@Ð@¸%Ð@Ñ@Ô@€IØ×ÒØØØØØAð ñ ô ð ð !ð ð ‰ˆˆxØ $ˆÔØ�Š�eÑÔÐÐðð r.   c                   ó   — e Zd Zdd„ZdS )ÚHackyCommitOperationAddr%   Nc                 óp   — t          | j        t          ¦  «        rt          | j        ¦  «        | _        d S d S rw   )rŒ   Úpath_or_fileobjr   rI   rÕ   s    r/   Ú__post_init__z%HackyCommitOperationAdd.__post_init__Ç  s:   € Ý�dÔ*­DÑ1Ô1ð 	=Ý#& tÔ';Ñ#<Ô#<ˆDÔ Ð Ð ð	=ð 	=r.   )r%   N)r­   r®   r¯   r9  r-   r.   r/   r6  r6  Æ  s(   € € € € € ð=ð =ð =ð =ð =ð =r.   r6  c                 ó  — | \  }}t          |j        |j        ¬¦  «        }|j                             d¦  «        5 }|                     d¦  «        d d…         }d d d ¦  «         n# 1 swxY w Y   |j        €t          d¦  «        ‚t          t           	                    |j        ¦  «        |j
        |¬¦  «        |_        |j        |_        |j        |_        |j        |_        |j        |_        |j        r|j        dk    rd|_        |S )N)rG   r8  r  i   z&sha256 must have been computed by now!)rË   r;   Úsampler¹   r.   )r6  rG   rP   r  ÚpeekrË   rˆ   r   ÚbytesÚfromhexr;   Úupload_inforÍ   r'  rá   r(  r*  r)  rÎ   Ú_is_uploadedr8  )rÑ   rW   rÒ   Ú	operationÚfiler;  s         r/   r  r  Ì  s7  € Ø�O€Eˆ8Ý'°UÔ5GÐY^ÔYhÐiÑiÔi€IØ	Œ×	Ò	˜dÑ	#Ô	#ð & tØ—’˜3‘”   Ô%ˆð&ð &ð &ñ &ô &ð &ð &ð &ð &ð &ð &øøøð &ð &ð &ð &à„ÐÝÐAÑBÔBÐBÝ&­e¯mªm¸H¼OÑ.LÔ.LÐS[ÔS`ÐioÐpÑpÔp€IÔØ%Ô1€IÔØ'Ô5€IÔØ$Ô/€IÔØ%Ô1€IÔØÔð ( Ô 4¸Ò =Ð =Ø$'ˆ	Ô!ØÐs   »A%Á%A)Á,A)rº   zqueue.Queue[JOB_ITEM_T]c                 ó"   —  | j         ¦   «         gS rw   ©Úget)rº   s    r/   r  r  â  s   € ØˆEŒI‰KŒKˆ=Ðr.   Únc                 óp   ‡ — ˆ fd„t          t           ‰ j        ¦   «         |¦  «        ¦  «        D ¦   «         S )Nc                 ó.   •— g | ]} ‰j         ¦   «         ‘ŒS r-   rD  )r9   r‡   rº   s     €r/   r}   z_get_n.<locals>.<listcomp>ç  s!   ø€ Ð>Ð>Ð>˜AˆIˆEŒI‰KŒKÐ>Ð>Ð>r.   )r˜   rÜ   r  )rº   rF  s   ` r/   r  r  æ  s7   ø€ Ø>Ð>Ð>Ð>¥¥s¨;¨5¬;©=¬=¸!Ñ'<Ô'<Ñ!=Ô!=Ð>Ñ>Ô>Ð>r.   Úreportc                 óJ  ‡— | dz  } t          j        ¦   «         j        Št          ˆfd„|                      ¦   «         D ¦   «         ¦  «        }t          |¦  «        D ]@}t          j                             d¦  «         t          j                             d¦  «         ŒAt          j                             | ¦  «         t          j                             d‰t          |                      ¦   «         d         ¦  «        z
  z  ¦  «         t          j         
                    ¦   «          dS )z÷Print a report, overwriting the previous lines.

    Since tqdm in using `sys.stderr` to (re-)write progress bars, we need to use `sys.stdout`
    to print the report.

    Note: works well only if no other process is writing to `sys.stdout`!
    ru   c              3   óB   •K  — | ]}t          |¦  «        ‰z  d z   V — ŒdS )r	   N)rC   )r9   ÚlineÚterminal_widths     €r/   r<   z#_print_overwrite.<locals>.<genexpr>÷  s3   øè è € ÐSÐS°t•3�t‘9”9 Ñ.°Ñ2ÐSÐSÐSÐSÐSÐSr.   z[Kz[Frß   r2   N)ÚshutilÚget_terminal_sizeÚcolumnsÚsumÚ
splitlinesr˜   ÚsysÚstdoutÚwriterC   Úflush)rI  Únb_linesr‡   rM  s      @r/   rž   rž   ê  s   ø€ ð ˆd�N€FåÔ-Ñ/Ô/Ô7€Nõ ÐSÐSÐSÐS¸v×?PÒ?PÑ?RÔ?RÐSÑSÔSÑSÔS€Hõ �8‰_Œ_ð #ð #ˆÝŒ
×Ò˜Ñ$Ô$Ð$ÝŒ
×Ò˜Ñ"Ô"Ð"Ð"õ „J×Ò�VÑÔÐÝ„J×Ò�S˜N­S°×1BÒ1BÑ1DÔ1DÀRÔ1HÑ-IÔ-IÑIÑJÑKÔKÐKÝ„J×ÒÑÔÐÐÐr.   )Lr°   Úloggingr�   rº   rN  rS  r„   rœ   r  r   Úpathlibr   r   Útypingr   r   Úurllib.parser   Ú_commit_apir
   r   r   Ú_local_folderr   r   r   r   Ú	constantsr   r   Úutilsr   r   r   r   Úutils._runtimer   Ú	utils.shar   Úhf_apir   Ú	getLoggerr­   rA   r
  r  r"   rú   rü   Ú__annotations__r”   r•   rD   rO   rS   rU   re   rI   rý   rª   ÚEnumr¬   Útuplerû   r—   r†   r  r  r  r  r	  r6  r  r  r  rž   r-   r.   r/   ú<module>rg     s®  ðð €€€€Ø €€€Ø 	€	€	€	Ø €€€Ø €€€Ø 
€
€
€
Ø Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø %Ð %Ð %Ð %Ð %Ð %Ð %Ð %Ø Ð Ð Ð Ð Ð à LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LÐ LØ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vÐ vØ 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ð 3Ø SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SÐ SØ ,Ð ,Ð ,Ð ,Ð ,Ð ,Ø "Ð "Ð "Ð "Ð "Ð "ð ð ØÐÐÐÐÐà	ˆÔ	˜8Ñ	$Ô	$€àÐ Ø!$Ð ØOÐOÐOÐ �4˜”9Ð OÐ OÑ OàÐ ØÐ ð Ð ØÐ ØÐ ØÐ ð]/¨Ð-AÔ(Bð ]/Àtð ]/ð ]/ð ]/ð ]/ðL  ØØ-1Ø.2Ø"ØØ ðC'ð C'ð C'Ø	ðC'àðC'ð �t‘ðC'ð
 ðC'ð �D‰jðC'ð �D‰[ðC'ð ˜”I ‘O dÑ*ðC'ð ˜#”Y ‘_ tÑ+ðC'ð �t‘ðC'ð ðC'ð ðC'ð C'ð C'ð C'ðVð ð ð ð �”	ñ ô ð ð Ð'Ð)@Ð@ÔA€
ðufð ufð ufð ufð ufñ ufô ufð ufðpl3Øðl3à	ðl3ð ðl3ð ð	l3ð
 ðl3ð l3ð l3ð l3ð^g(Ð 1ð g(°e¸IÀtÈJÔGWÐ<WÔ6XÐ[_Ñ6_ð g(ð g(ð g(ð g(ð^˜*ð ¨ð ð ð ð ð˜D Ô,ð °7ð ÀSð ÐUXð Ðdgð Ðlpð ð ð ð ð,˜$˜zÔ*ð °ð À3ð ÐSVð Ðbeð Ðjnð ð ð ð ð�4˜
Ô#ð ¨'ð ¸Cð ÈCð Ð[^ð Ðcgð ð ð ð ð(=ð =ð =ð =ð =Ð0ñ =ô =ð =ð ð Ð0Gð ð ð ð ð,Ð-ð °$°zÔ2Bð ð ð ð ð?Ð+ð ?°ð ?¸¸ZÔ8Hð ?ð ?ð ?ð ?ð˜Sð  Tð ð ð ð ð ð r.   