}tjh.dZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z ddl m Z mZddlmZmZmZddlmZdd lmZmZmZmZmZmZdd lmZdd lmZdd l m!Z!dd l"m#Z#ddl$m%Z%ddl&m'Z'm(Z(ddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6ddl7m8Z8m9Z9ddl:m;Z;mZ?de@de@dzdeAe@e@dzffdZBdeCe@dzdede@dzdeCedzfdZDdZEee@e8dfZFee@dze9dfZGee@dze9de.e fZHeeCe@dze9d!d"d#fZIee@dze9d$d%fZJee@dze9d&fZKeeLe9d'd(d)fZMee@dze9d*fZNee@dze9d+d,fZOeeCePdze9d-d.fZQeeLe9d/d0fZReeCe@dze9d1d2fZSee@dze9d3d4d5fZTeeLdze9d6fZUeeLdze9d7fZVee@e8d8fZWee@e8d9fZXeeCe@dze8d:fZYeeCe@e8d;fZZee@e8d<fZ[eeCe@dze8d=fZ\ee@e8d>fZ]eeCe@dze9d?d@dAfZ^e6dBZ_dedCeddfdDZ`e_adEdFdGigdHI ddKeFdLeZdMe+dNe-dOeJdPeIdQe^dRe*dSe,dTeHdUeKdVeMdWeQdXeRdYeOdeNdZe1ddf$d[Zbe_ad\gd]^ dde[d_eeLe9d`dadbfdceePdze9dddedffdeNdZe1ddf dgZcdhede@e@fdieCeAe@e@e@fdeLfdjZedkePddfdlZfde@dmeede@efdneCe@deeAeLe@eCeCe@ePzffdoZge_adpdqg^ ddre\deNdZe1ddfdsZhe_adtgdu^ ddweeLe9dxdydzfd{eeCe@dze9d|e.ed}~fdPeeCe@dze9d!d"dfdOee@dze9d$dfdeePe9ddfdeNdZe1deeCe@dze9d`ddfddfdZie_addg^ddZje_addg^ ddreeCe@e8dfdeNdZe1ddfdZke_addg^ dde[deNdZe1ddfdZle_adgd^ ddreeCe@e8dfdUee@dze9dfdeNdZe1ddf dZme_adgd^ dde[dOeJdPeIdeeLe9ddfdeNdZe1ddfdZne_adXgd^ dde[de0de/deNdZe1ddf dZoe6dZpe_qepdepadEdFdGigdI ddeXdeYdKeGdTeHdMe+dNe-dOeJdPeIdQe^dRe*dSe,dUeKdVeMdWeQdXeRdYeOdeNdZe1deSdeTddf*dZre6dZse_qesdesadEdFdGidgI ddeWdKeFdLeZdeUdeVdMe+dNe-dOeJdPeIdQe^dRe*dSe,dTeHdUeKdWeQdYeOdeNdZe1ddf&dZtesadtdg^ ddweeLe9dxdydfdeNdZe1deeCe@dze9d`ddfddf dZuesaddg^ ddeeCe@e8dfdeNdZe1ddfdZvesaddg^ dde]deNdZe1ddfdZwesaddg^ dde]deNdZe1ddfdZxesaddg^ dde]deNdZe1ddfdZyesaddg^ dde]deNdZe1ddfdĄZzesadgdŢ^ dde]dOeJdPeIdeeLe9ddƬfdeNdZe1ddfdDŽZ{e6dȬZ|esqe|de|adEdFdGiddgI ddeWdeXdeYdeUdeVdKeGdTeHdMe+dNe-dOeJdPeIdQe^dRe*dSe,dUeKdWeQdYeOdeNdZe1deSdeTddf,d˄Z}dede@efdede@e@fdzdede@effd̈́Z~dOe@dzdPeCe@dzdeLfd΄ZdddeCe@dzdOe@dzdede@e@fdzfdτZdeCeCe@ePzdeCe@de@fd҄ZedӦZdeededeefdedddfd؄ZdejjdedeefdeCeddeefdۄZdS)z@Contains commands to interact with jobs on the Hugging Face Hub.N)CallableIterable)fnmatch)Path)EmptyQueue) AnnotatedAnyTypeVar)urlsplit)HfApi JobHardwareJobInfoJobStageVolume constants)TERMINAL_JOB_STAGES)CLIError)logging) _format_size) _split_mount)format_durationparse_duration) EnvFileOptEnvOptSecretsFileOpt SecretsOpt SoftChoice SshDryRunOptSshIdentityFileOptTokenOptexec_ssh get_hf_api parse_env_map parse_volumes typer_factory)ArgumentOption)_dataclass_to_dictoutjob_id namespacereturnc0|std|ddkrtd|dd|vr||fS|dd\}}|r|std|d|||krtd|d|d ||fS) a&Extract namespace from job_id if provided in 'namespace/job_id' format. Allows users to pass job IDs copied from the Hub UI (e.g. 'username/job_id') instead of only bare job IDs. If the namespace is also provided explicitly via --namespace and conflicts, a CLIError is raised. zJob ID cannot be empty./rz`. Requires an SSH public key registered on https://huggingface.co/settings/keys.z--withz%Run with the given packages installedz-pz--pythonz5The Python interpreter to use for the run environmentz!Suspend (pause) the scheduled Jobz8Allow multiple instances of this Job to run concurrentlyzWOne of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.z$UV script to run (local file or URL)zArguments for the scriptzThe command to run.zJob ID (or 'namespace/job_id')zJob IDs (or 'namespace/job_id')z2Scheduled Job ID (or 'namespace/scheduled_job_id')z-vz--volumeaMount one or more volumes. Format: hf://[TYPE/]SOURCE:/MOUNT_PATH[:ro|:rw] or LOCAL_DIR:/MOUNT_PATH[:ro|:rw]. TYPE is one of: models, datasets, spaces, buckets. TYPE defaults to models if omitted. models, datasets and spaces are always mounted read-only. buckets are read+write by default. A local directory source is first synced to a bucket and mounted read-only by default. E.g. -v hf://datasets/org/ds:/data or -v hf://buckets/org/b:/mnt:ro or -v ./inputs:/inputszRun and manage Jobs on the Hub.jobc||j|jjdD]}t j|||j|jj}|jjtj krD|jj rd|jj nd}td|jd|jjd|t jd|jd d S) zcStream Job logs until the Job ends, then fail the command if the Job did not complete successfully.T)r,r-followr,r-z: zJob z finished with stage 'r1z completedN) fetch_job_logsidownernamer+text wait_for_jobstatusstager COMPLETEDmessager)r:rPlogfinalr^s r6_stream_logs_and_check_statusra>s!!39>RV!WW    CFcin  E EE |X///161EM-u|+---2\eh\\el>P\\SZ\\]]]H (EH ( ( ()))))r8runignore_unknown_optionsT)zFhf jobs run --name hello-world python:3.12 python -c 'print("Hello!")'z1hf jobs run --detach python:3.12 python script.pyz3hf jobs run -e FOO=foo python:3.12 python script.pyz;hf jobs run --secrets HF_TOKEN python:3.12 python script.pyz^hf jobs run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt python:3.12 python script.py)context_settingsexamplesFimagecommandenvsecretsrXlabelrLenv_file secrets_fileflavortimeoutdetachexposesshresource_group_idtokenct||}t||}t|}|||||t||t |||| | | | || }t jd|j|jpi d|j t||sI|jpi d}t j d|d |j jd |jd t|jjt$r@d d |jjD}t j d|t|jjt*r*t j d|j jd |jd| rK|j jd |j}t j d|d|dt j d|ddSt-||dS)z Run a Job.rsrXr:r-) rfrgrhrilabelsr9rmrnrprqrrr- Job startedrXrVrXurlJob auto-named '('. Pass `--name` or run `hf jobs labels r0 --name NAME` to rename it. c3 K|] }d|V dSz N.0r{s r6 zjobs_run..(FFcFFFFFFr8SExposed ports are reachable at (requires an HF token with read access to the job): Use `hf jobs ssh &` to open an SSH session into the job.Use `hf jobs logs -f &` to stream logs, or `hf jobs inspect ` to check status.Use `hf jobs wait ` to block until it finishes.N)r%r$run_job_parse_labels_maprMr+rIrVrxgetr{_has_explicit_namerErWrX isinstancer[ expose_urlslistjoinssh_urlstrra)rfrgrhrirXrjrLrkrlrmrnrorprqrrr-rsenv_map secrets_mapr:rP auto_nameurlsjob_refs r6jobs_runrJsV>C**G66K 5 ! ! !C ++  T222+FyQQQ +   CJ}sz/?R.D.DV.L.LRURYZZZZ dE * * Z%2**622   Ty T T"y~ T T03 T T T   #*($//pyyFFsz/EFFFFF nhlnnooo#*$c**f dSY^ddcfdddeee Y^..cf.. sssX_sssttt LgLLLMMM!#s+++++r8logs)zhf jobs logs zhf jobs logs -f zhf jobs logs --tail 20 z#hf jobs logs -f --tail 100 )rerRz-fz--followzqFollow log output (stream until the job completes). Without this flag, only currently available logs are printed.tailz-nz--tailzvNumber of lines to show from the end of the logs. When combined with --follow, starts streaming from the last N lines.ct||\}}t|}|||||}|D]}tj||r%|r|d|n|}tjd|ddSdS)aFetch the logs of a Job. By default, prints currently available logs and exits (non-blocking). Use --follow/-f to stream logs in real-time until the job completes. Use --tail/-n to limit the number of lines returned (server-side when supported). Note: following exits when the log stream ends, regardless of whether the Job succeeded or failed. Run `hf jobs inspect ` to check the final status. ru)r,r-rRrr0z#Stream ended. Run `hf jobs inspect z6` to check the final status (e.g. COMPLETED or ERROR).N)r7r$rUr+rYrE) r,rRrr-rsr:rr_rs r6 jobs_logsrsN5VYGGFI 5 ! ! !C   VyVZ  [ [D   x-6BY)))))F vwvvvwwwwwxxr8job_propertiesfiltersc|D]n\}}}||}| |dkr$dSt||}|dkr|r|dkr|rdSodS)z5Check if scheduled job matches all specified filters.N!=F=T)rrlower)rrkeyop_strpatternvaluematchs r6_matches_filtersrs 'VW""3'' =~~55 w}}77 cMM%MVt^^^55 4r8ncTd}d}t|D]}t||dS)Nzz)end)rangeprint)rLINE_UP LINE_CLEARis r6 _clear_linersAGJ 1XX'' g:&&&&&''r8metrics_stream table_headersc #K|D]}||ddt|ddz dtd|dz|dz d dt|dd t|dd t|d d t|ddg}|drt|dtr|gdgt |zgt |ddz zz}t |t |dD]s\}}|d|}||ddtd|dz|dz d dt|dd t|dd gz }tn)|dgt |t |z zz }|g}d||fVd|gfVdS)N cpu_usage_pct%cpu_millicoresg@@rdmemory_used_bytesmemory_total_byteszB / Brx_bpszbps / tx_bpsbpsgpusrT utilizationN/AFT)roundrrdictlenzipsorted)r,rrmetricsrowrowsgpu_idgpus r6_get_jobs_stats_rowsrsA""" ' * * * '*+f4a 8 8S7#677'BV:WWYZ[[ ^ ^ ^G$7899 m m|GThLi?j?j m m mGH-.. Z Zl78CT6U6U Z Z Z   6? z'&/4@@ 5RD3s88O,GFO0D0Dq0HIID"4)@)@AA   Vfof-=),,,S3':#;;cBV>WWYZ[[^^^#C(;$<==mmjob_idsc6 |1g}|D]*}t|\}||+|}t|  d| d D}t |dkrt jddSgdtj t |5}i|D]"}|g}|d d dDz }|g|<#tj }fd D}tt|  fd |D} t|t| D]\} }} | r|dn| |<tj } | |z t"krMt%dt |zfdD}tt| | } ddddS#1swxYwYdS)z7Fetch the resource usage statistics and metrics of JobsNrurXcJg|] }|jr |jjnddv|j!S)UNKNOWN)RUNNINGUPDATING)r[r\rVrrPs r6 zjobs_stats..sA   $'J=   IBYYY FYYYr8r-rzNo running jobs found) zJOB IDzCPU %zNUM CPUzMEM %z MEM USAGEzNET I/Oz GPU UTIL %z GPU MEM %z GPU MEM USAGEc&g|]}d|vsd|vrdndS)r0USAGEz-- / --z--r)rheaders r6rzjobs_stats..s/pppTZ#--7f3D3DII4pppr8rc*g|]}|D]}|Srrrr,rrows_per_job_ids r6rzjobs_stats..s,[[[f?SYCZ[[Cc[[[[r8)headerscFg|]}||dS)rS)r,rr)fetch_job_metrics)rr,r:r-rs r6rzjobs_stats.."sO    !"%"7"7vQZ"7"["[!.     r8) kwargs_listrc*g|]}|D]}|Srrrs r6rzjobs_stats..2s,cccf?[aKbccCcccccr8)r7rHr$whoami list_jobsrr+rYmultiprocessingpool ThreadPooltimer _tabulateiflatmap_unorderedrpopSTATS_UPDATE_MIN_INTERVALr)rr-rs parsed_idsr,rrlast_update_time total_rowsrdonernowr:rrs ` @@@r6 jobs_statsrs  & &F  , ,F$*8C pp^klmlnln^oppp pC'*eOF # #9;;[[[[[[[  i M:::;;;      "    #5T;O]h"i"i"i ' ' D&$ /##FD1111*.')++C%%)BBBAJ/000ccccccc i MBBBCCC#&  ')''''''''''''''''''sD!HHHzlist | ls | ps)z hf jobs lsz hf jobs ls -az&hf jobs ls --status running,schedulingzhf jobs ls --name training-v2z+hf jobs ls --label env=prod --label team=mlz%hf jobs ls --all --label hf-sandbox=1rallz-az--allzWShow all Jobs (default shows running and scheduling). Cannot be combined with --status.r[z--statuszfOnly show Jobs with the given status. Comma-separated or repeated, e.g. `--status running,scheduling`.)rOrNz{Only show Jobs with the given `key=value` label. Repeat to require several labels, e.g. `--label env=prod --label team=ml`.zFOnly show Jobs with the given name (shortcut for `--label name=NAME`).limitz--limitzCMaximum number of Jobs to display. Set to 0 to show all (no limit).filterz--filterz2(Deprecated) Use `--status` and `--label` instead.ct|}|rtjd|d|r|rtdg} |pgD]4} | d| dD5| r| } n'|rd} n"t jjt j jg} i} |pgD]6} d| vrtd | d | d\}} | | |<7|d | vrtd || d <| || | pd }d}|dkrEttj ||dz}t||kr d}|d|}nt|}g}|D]d}t|}|dpi}|dpg}|dd|d<|dpid pd|d <|dpd|d<|rd|nd|d<|dr$|dddddnd|d<|d pid!d"|d <t'|d#|d$<||ftj|gd%d&|rtjd'|d(|st| s| rXd)gd*| Dd+| D}tjd,|dS|stjd-dSdSdS).zList Jobs. Use `--status` to filter by status (see [`JobStage`] for possible values) and `--label` to filter by `key=value` labels. A Job must match every filter to be listed. ruzIgnoring filter 'zc'. `-f`/`--filter` is deprecated and will be removed in a future release. Use `--status`/`--label`.z0`-a`/`--all` cannot be combined with `--status`.c3fK|],}||V-dSN)strip)rparts r6rzjobs_ps..s7VVTVDJJLLVVVVVVr8,NrzInvalid label filter 'z"': must be in the form 'key=value'rXz6Cannot filter by both `--name` and `--label name=...`.)r-r[rxFrrT durationsrgrVrTr,rxr docker_image image/spacer> created_atTcreatedr[r\r running_secsruntime)r,rXrrgrr[r rid_keyzOutput truncated to z4 Jobs. Use `--limit 0` to show all (or `--limit N`)., c3 K|] }d|V dS)zstatus=Nr)rss r6rzjobs_ps..s(77Q=Q==777777r8c3,K|]\}}d|d|VdS)zlabel=rNr)rkvs r6rzjobs_ps..s::g:gPTPQST;KA;K;K;K;K:g:g:g:g:g:gr8zNo jobs matched filters: zHNo running jobs. Use `-a`/`--all` to include finished (and failed) jobs.)r$r+warningrrAr3rrr SCHEDULINGrr itertoolsislicerr*rrreplacerrHtablerEitemsrY)rr[rjrXrr-rsrr: raw_statusesrserver_statusesrxitemr jobs_iter truncatedjobs job_itemsrPjob_itemrcmd filters_msgs r6jobs_psr$7snD 5 ! ! !C    p p p p    KvKIJJJ!L2WWVVU[[5E5EVVVVVVVN& N#+183F3LM F  d??\D\\\]] ]ZZ__ Us   V  STT Tv /RXR`\` aaII qyyI$Y ::;; t99u  I">"G%/2=chhsmmmOW||\hOiOith|4SbS9AA#sKKKot&ll844:??SS-immN.K.KLL""""I\\\ e ccccddd a  a6 a))h77,777h:g:gX^XdXdXfXf:g:g:ghK H>>> ? ? ? ? ? a H_ ` ` ` ` `aa  a ar8hardwarezhf jobs hardwarec t}|}g}|D]}d}|jr)|jjd|jjd|jjd}|jr d|jdnd}|jrd|jdzd nd}||j|j |j |j |j |||d tj|tjd d S) z(List available hardware options for JobsrTzx  ()$z.4ffree<z.2f)rXz pretty namecpuramstorage acceleratorzcost/minz cost/hourzLUse `hf jobs run --flavor ...` to request a specific hardware flavor.N)r$list_jobs_hardwarer/quantitymodelvram unit_cost_usdrHrX pretty_namer,r-ephemeral_storager+rrE)r: hardware_listrhwaccelerator_infocost_min cost_hours r6 jobs_hardwarer<s+ ,,C**,,M E   > l"$."9kkR^=QkkUWUcUhkkk 131AM-r'----v797GS3(2-3333V  !~vv//$&   IeH [\\\\\r8inspectzhf jobs inspect z*Job IDs to inspect (or 'namespace/job_id')cg}|D]*}t|\}||+|}t|fd|D}tjd|DddS)z0Display detailed information on one or more Jobsruc>g|]}|S)rS) inspect_job)rr,r:r-s r6rz jobs_inspect..s) V V VFCOO6YO ? ? V V Vr8cTg|]%}tt||j&S)rx) _surface_namer*rxrs r6rz jobs_inspect..s0YYYS}/44SZHHHYYYr8rVr Nr7rHr$r+r)rr-rsrr,rr:s ` @r6 jobs_inspectrFsJ""8KK &!!!!G 5 ! ! !C V V V V Vg V V VDIYYTXYYYbfggggggr8cancelzhf jobs cancel ct||\}}t|}|||tjd|dS)z Cancel a JobrurSz Job cancelledrVN)r7r$ cancel_jobr+rI)r,r-rsr:s r6 jobs_cancelrKsY5VYGGFI 5 ! ! !CNN&IN666J6******r8wait)zhf jobs wait z"hf jobs wait z"hf jobs ls -q | xargs hf jobs waitz,Job IDs to wait for (or 'namespace/job_id').zTMax time to wait: int with s (seconds, default), m (minutes), h (hours) or d (days).c g}t}|D]?}t||\}}||||@t |dkrCt dddt|tDz| }|t|nd} t|} tj dt |d } | || | } n7#t$r*| d t d |d dwxYw| t | dtjd| Dd| D} | rOt t | dt | ddd| DzdS)aWait for one or more Jobs to reach a terminal state. Blocks until every Job has finished, then exits with code 0 if all Jobs completed successfully, or a non-zero exit code if any Job was canceled, errored or deleted. All Jobs must belong to the same namespace. rz0All Job IDs must be in the same namespace, got: r c34K|]}t|VdSr)r)rnss r6rzjobs_wait..2s(FFBBFFFFFFr8)rNruz Waiting for z Job(s) to finish...)rnr-z Timed out.zTimed out after z waiting for Job(s) to finish.z Job(s) finished.cfg|].}|jt|jj|jjd/S))rVr\r^)rVrr[r\r^rs r6rzjobs_wait..@s7mmmadcfs3:+;'<'<I[\\mmmr8cFg|]}|jjtjk|Sr)r[r\rr]rs r6rzjobs_wait..As* L L LcSZ%59K%K%Kc%K%K%Kr8z of z' Job(s) did not complete successfully: c3DK|]}|jd|jjdVdS)r'r(N)rVr[r\rs r6rzjobs_wait..Es:KKS3688SZ%5888KKKKKKr8)setr7rHaddrrrrrrrr$r+r[rZ TimeoutErrorrr)rrnr-rsr namespacesr, parsed_idparsed_namespace timeout_secsr:r[rfaileds r6 jobs_waitr[ s>>JJ))&B69&U&U# #)$$$'(((( : >iiFF&*E*E*EFFFFF G     I.5.A>'***tL 5 ! ! !C ZLs:LLL M MF] LIVV ]]] L!!!Q'QQQRRX\\] KK3t99///000Immhlmmmnnn L LT L L LF  6{{ R RD R R RiiKKFKKKKK L     s D4Erx)z*hf jobs labels --name training-v2z8hf jobs labels --label env=prod --label team=mlzhf jobs labels --clearclearz--clearzRemove all labels from the job.c|s||std|s||rtdt||\}}t|}|)|s'|||jpi}i|d|i}nt ||pi}||||} tjd | j | d dS) zfUpdate labels on a Job. Passing --label replaces all existing labels; passing --name alone keeps them.NePlease set a name with --name or at least one label with --label. To remove all labels, pass --clear.lCannot set a name or labels and clear them at the same time. Please use --name/--label or --clear, not both.rurSrXrv)r,rxr-Labels updatedrVrX) rr7r$r@rxrupdate_job_labelsr+rIrVr) r,rXrjr\r-rsr:current_labelsrxrPs r6 jobs_labelsrdIs"  T\%\ s     !u! z   5VYGGFI 5 ! ! !C )LLSYWY1N1FD11"5t444:  vf  R RCJCFF1C1CDDDDDDr8)zhf jobs ssh zhf jobs ssh --dry-runz)hf jobs ssh -i ~/.ssh/id_ed25519 identity_filedry_runcJt||\}}t|}|||}|jjt d|jjtvr%t d|jd|jjd|jjtj krtjd|jd |jjd }| |j|tj g }|jjtj kr:| d t d|jd |jjd| dt|jj}t|jd|j|j||dS)a SSH into a running Job. If the Job is not yet running, waits until it reaches the RUNNING state before connecting. Requires the Job to be started with SSH enabled (`hf jobs run --ssh ...`) and your SSH public key to be registered at https://huggingface.co/settings/keys. rurSNz[SSH is not enabled on this job. Start a job with SSH support using `hf jobs run --ssh ...`.zCannot SSH into job 'z%': job has already finished (stage: 'z').zWaiting for job 'z' to be running (stage: 'z')...)r,r-stagesz Job finished.z1': job finished before reaching RUNNING (stage: 'zJob is running.@)portrerf)r7r$r@r[rrr\rrVrrr+rZrr r#usernamehostnamerj) r,rerfr-rsr:rPr[rs r6jobs_sshrmns*5VYGGFI 5 ! ! !C //9/ = =C z!tuuu z...qsvqq\_\f\lqqqrrr z8+++hhhQTQ[Qahhhiicf 8K[J\]] : x/ / / KK ( ( (vvvadakaqvvv   %&&&sz)**G  00g.00 \# r8zFRun UV scripts (Python with inline dependencies) on HF infrastructure.uvrv)z,hf jobs uv run --name my-script my_script.pyz$hf jobs uv run --detach my_script.pyz1hf jobs uv run ml_training.py --flavor a10g-smallz+hf jobs uv run --with transformers train.pyzNhf jobs uv run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt script.pyscript script_argswith_pythonc$t|| }t|| }t|}|||pg|||||t||t ||||| | |||}t jd|j|jpi d|j t||sI|jpi d}t j d|d |j jd |jd t|jjt$r@d d |jjD}t j d|t|jjt*r*t j d|j jd |jd| rK|j jd |j}t j d|d|dt j d|ddSt-||dS)8Run a UV script (local file or URL) on HF infrastructurerurvrw)rorp dependenciesrrrfrhrirxr9rmrnrprqrrr-ryrXrzr|r}r0r~rc3 K|] }d|V dSrrrs r6rzjobs_uv_run..rr8rrrrrrrrN)r%r$ run_uv_jobrrMr+rIrVrxrr{rrErWrXrr[rrrrrra)rorprfrmrhrirXrjrLrkrlrnrorprqrrr-rsrqrrrrr:rPrrrs r6 jobs_uv_runrxseDC**G66K 5 ! ! !C ..%2  T222+FyQQQ +   C"J}sz/?R.D.DV.L.LRURYZZZZ dE * * Z%2**622   Ty T T"y~ T T03 T T T   #*($//pyyFFsz/EFFFFF nhlnnooo#*$c**f dSY^ddcfdddeee Y^..cf.. sssX_sssttt LgLLLMMM!#s+++++r8z,Create and manage scheduled Jobs on the Hub. scheduledzRhf jobs scheduled run "0 0 * * *" --name daily-script python:3.12 python script.pyschedulesuspend concurrencycpt|| }t|| }t|}||||||||t||t | ||| | |||}t jd|j|jj pi dt||sN|jj pi d}t j d|d |j jd |jd t j d |j jd |jd dS)zSchedule a Job.rurvrw)rfrgrzr{r|rhrirxr9rmrnrprrr-Scheduled Job createdrXraScheduled Job auto-named '2'. Pass `--name` or run `hf jobs scheduled labels r0r~Use `hf jobs scheduled inspect ` to view its details.N)r%r$create_scheduled_jobrrMr+rIrVjob_specrxrrrErWrX)rzrfrgr{r|rhrirXrjrLrkrlrmrnrprrr-rsrrr: scheduled_jobrs r6 scheduled_runrs4C**G66K 5 ! ! !C,,  T222+FyQQQ+-M J&=+;=CYC`CfdfBkBklrBsBstttt dE * * "+28b==fEE   r r r)6)<)A r rDQDT r r r   H r}/B/G r r-JZ r r rsssssr8zhf jobs scheduled lsz1Show all scheduled Jobs (default hides suspended)z>Filter output based on conditions provided (format: key=value)ct|}||}g}|pgD]}d|vrz|dd\}} |dr d} |dd}nd} ||| | ft jd |d g} |D]} | jpd } |s| r| j j pd }| j j pg}|rd |nd }| j j pidpd }| j||t!| |d}t#||s| | g}| D]~}t%|}|dpi}|dpi}|d}|dpg}|dpidpd |d<|dpd |d<|rd |nd |d<|r9|dr$|ddddd nd |d<|dr$|ddddd nd |d<|dpd |d<||t j|gdd |s8|r6d! d"|D}t jd#||r@|d$d}t jd%|d&t jd'|d(dSdS))zList scheduled Jobsrurrr!rNz Ignoring invalid filter format 'z'. Use key=value format.Frr>rX)rVrXrfr{rgrr[last_jobrgrxrratrrlast_runnext_job_run_atnext_runr{)rVrXrzrrgrrr{rVr r c3.K|]\}}}|||VdSrr)rrors r6rzscheduled_ps..zs4DDAq1 a  DDDDDDr8z#No scheduled jobs matched filters: rrz(` to view details about a scheduled job.zUse `hf jobs scheduled trigger z)` to trigger a scheduled job immediately.)r$list_scheduled_jobsr3endswithrHrr+rr{rrrgrrxrrVrrr*rrrYrE)rr-rsrr:scheduled_jobsrfrrop filtered_jobsrr{image_or_spacer" command_strjob_namepropsrsjrr status_dictrr# first_item_ids r6 scheduled_psr's&, 5 ! ! !C,,y,AAN*,G \r X X !88aJC||C   #2#h NNCIIKKU[[]]; < < < < KV1VVV W W W WM',, '05 w  &/<E$,2'*5chhsmmm !*17R<::Ce]+.9#((3---EYDLnQYQ]Q]^bQcQcn8D>#2#.66sC@@@inZEP__UfEgEg rK) *3B3 / 7 7S A A Amr Z((9--6Y TI ggg FWFiiDDGDDDDD  D{DDEEE ma  j=jjjkkk k=kkklllllmmr8zhf jobs scheduled inspect scheduled_job_idsz>Scheduled Job IDs to inspect (or 'namespace/scheduled_job_id')cg}|D]*}t|\}||+|}t|fd|D}tjd|DddS)z:Display detailed information on one or more scheduled Jobsruc>g|]}|S)scheduled_job_idr-)inspect_scheduled_job)rrr:r-s r6rz%scheduled_inspect..s=  !!3Cy!YYr8c^g|]*}tt||jj+SrB)rCr*rrx)rrs r6rz%scheduled_inspect..s4iiiTV}/33BKrct||\}}t|}|||tjd|dS)zDelete a scheduled Job.rurzScheduled Job deletedrIN)r7r$delete_scheduled_jobr+rIrr-rsr:s r6scheduled_deleterb#??OQZ"["[i 5 ! ! !C.>)TTTJ&+;<<<<<ct||\}}t|}|||tjd|tjd|ddS)z Suspend (pause) a scheduled Job.rurzScheduled Job suspendedrIzUse `hf jobs scheduled resume z` to resume it.N)r7r$suspend_scheduled_jobr+rIrErs r6scheduled_suspendrs#??OQZ"["[i 5 ! ! !C/?9UUUJ(-=>>>>H O.> O O OPPPPPr8resumezhf jobs scheduled resume ct||\}}t|}|||tjd|dS)z!Resume (unpause) a scheduled Job.rurzScheduled Job resumedrIN)r7r$resume_scheduled_jobr+rIrs r6scheduled_resumerrr8triggerzhf jobs scheduled trigger ct||\}}t|}|||}tjd|j|jtjd|jj d|jddS) zJTrigger a scheduled Job to run immediately (does not change the schedule).rurzScheduled Job triggered)rVr{rr0z` to stream logs.N) r7r$trigger_scheduled_jobr+rIrVr{rErWrX)rr-rsr:rPs r6scheduled_triggerrs#??OQZ"["[i 5 ! ! !C # #5EQZ # [ [CJ(SVAAAAH OSY^ O Ocf O O OPPPPPr8)z1hf jobs scheduled labels --name daily-scriptz>hf jobs scheduled labels --label env=prod --label team=mlz%hf jobs scheduled labels --clearz)Remove all labels from the scheduled job.c|s||std|s||rtdt||\}}t|}|.|s,|||jjpi}i|d|i}nt ||pi}||||} tj d | j | d dS) zpUpdate labels on a scheduled Job. Passing --label replaces all existing labels; passing --name alone keeps them.Nr^r_rurrXrv)rrxr-r`ra) rr7r$rrrxrupdate_scheduled_job_labelsr+rIrVr) rrXrjr\r-rsr:rcrxrs r6scheduled_labelsrs0"  T\%\ s     !u! z   #??OQZ"["[i 5 ! ! !C   % %7GS\ % ] ] f m sqs 2N1FD11"5t444:33)&I4MJM$46::f;M;MNNNNNNr8z)Schedule UV scripts on HF infrastructure.zBhf jobs scheduled uv run "0 0 * * *" --name daily-script script.pyz.Ps:"["[t4::c1#5#5a#8F#B"["["["["["[r8)any)rXrjs r6rrNs. t  [s"["[u{XZ"["["[[[[r8c|s|dSi}|pgD](}d|vr|ddn|df\}}|||<)|d|vrtd||d<|S)aParse label key-value pairs from CLI arguments. Args: labels: List of label strings in KEY=VALUE format. If KEY only, then VALUE is set to empty string. Returns: Dictionary mapping label keys to values, or None if no labels or name provided. NrrrTrXz4--name and --label name=... cannot both be provided.)r3r)rxrX labels_map label_varrrs r6rrSs dlt!#J\r  03y0@0@Y__S!,,,yRTo U 3  Z  QRR R! 6 r8rrcdtg||RD}ttjjt |dz}t |t |z|kr|t|}||xxdzcc<t |t |z|kr.|t |z t |z ||z||<t |t |z|kdt |zj|}g}| |j|| |jd|D|D]9}dt||D}| |j|:d |S)z Inspired by: - stackoverflow.com/a/8356620/593036 - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data c@g|]}td|DS)c3NK|] }tt|V!dSrrr)rxs r6rz'_tabulate...ps.//ac#a&&kk//////r8)max)rcols r6rz_tabulate..ps/OOOC#//3/////OOOr8 rz{{:{}}} cg|]}d|zS)-r)rws r6rz_tabulate..zs$A$A$AS1W$A$A$Ar8cg|]Q\}}tt||krt|d|dz dznt|RS)Nz...r)rr col_widths r6rz_tabulate..|sf   9033q66{{Y/F/FCFF?Y]? #e + +CPQFF   r8r) rrshutilget_terminal_sizecolumnsrsumindexformatrHr) rr col_widthsterminal_widthcol_to_minimize row_formatlinesrrow_format_argss r6rrisPO3;N;Ng;N;N;NOOOJ133;S\\B=NOON g,,Z (> 9 9$**3z??;;?###)### w<<#j// )^ ; ;*83z??*JSQX\\*Y\fgv\w*wJ ' g,,Z (> 9 9 4*s7||+3Z@J E LL""G,--- LL""$A$Aj$A$A$ABCCC::  #C 4 4     &Z&89999 99U  r8rqueuefunc.kwargscF|di|D]}||dS)Nr)put)rrrrIs r6_write_generator_to_queuers<$.... &r8rrc#Ktfd|D} dVn?#t$r2td|DrrYnYnwxYwZn#t $rYnwxYw d|DdS#t j$rYdSwxYw# d|Dw#t j$rYwwxYwxYw)a; Takes a function that returns an iterable of items, and run it in parallel using threads to return the flattened iterable of items as they arrive. This is inspired by those three `map()` variants, and is the mix of all three: * `imap()`: like `map()` but returns an iterable instead of a list of results * `imap_unordered()`: like `imap()` but the output is sorted by time of arrival * `flatmap()`: like `map()` but given a function which returns a list, `flatmap()` returns the flattened list that is the concatenation of all the output lists cLg|] }t|f!Sr) apply_asyncr)rrrrrs r6rz&iflatmap_unordered..s2rrr\bT%%&?%vAVWWrrrr8T皙?rnc3>K|]}|VdSr)readyr async_results r6rz%iflatmap_unordered..s.NN |))++NNNNNNr8c:g|]}|dS)rrrrs r6rz&iflatmap_unordered..s) N N N \  d  + + N N Nr8N)rrrremptyKeyboardInterruptrrU)rrr async_resultsrs`` @r6rrsfggErrrrrrfqrrrM  iii------   NN NNNNNSXS^S^S`S`E           N N N N N N N N+    DD   N N N N N N N+    D sBAB7A<7B;A<<BB6 B B6 BB6 B B32B36C8 CCCCCC)NNNNNNNNNFNFNNN)FNNN)NNN)FNNNrNNN)r.N)NN)NNFNN)NFNN)NNNNNNNNNNNFNFNNNNN)NNNNNNNNNNNNNNN)NNNNNNNNNNNNNNNNNNN)__doc__rrmultiprocessing.poolrrcollections.abcrrrpathlibrrrrtypingr r r urllib.parser huggingface_hubr rrrrrhuggingface_hub._jobs_apirhuggingface_hub.errorsrhuggingface_hub.utilsr$huggingface_hub.utils._cache_managerrhuggingface_hub.utils._hf_urisrhuggingface_hub.utils._parsingrr _cli_utilsrrrrrr r!r"r#r$r%r&r' _frameworkr(r)_outputr*r+ get_logger__name__loggerrtupler7rrMrImageArgImageOpt FlavorOpt LabelsOptNameOpt TimeoutOptbool DetachOpt NamespaceOptResourceGroupIdOptint ExposeOpt SshEnabledOptWithOpt PythonOpt SuspendOptConcurrencyOpt ScheduleArg ScriptArg ScriptArgsArg CommandArgJobIdArg JobIdsArgScheduledJobIdArg JobVolumesOptjobs_clirargrrrrrrrr$r<rFrKr[rdrmuv_app add_grouprx scheduled_apprrrrrrrrscheduled_uv_apprrCrrrrrrrrrr8r6r%s1GF ........**********!!!!!!TTTTTTTTTTTTTTTT999999++++++))))))======777777JJJJJJJJ)(((((((,,,,,,,,  H % %..t.cSVY]S]oH^....8) #Y )',)9X ]a    '3'4'''' !)$sCx.!9JNs) eD#tDsO44568 '%=$>??"A'A' A'A' A' A'A'A'@?A'H   &     " aFaFa    j     Fa  S D !z(++y     Fa"  S D  O     #Fa2  d  Y     3Fa@   V     AFaNOFaP QFaR  S D  E     SFab cFaFaFa  FaR *(:';<<]]]=<]6 )'A&BCC#hh  S =    hh h hhhDCh* (&?%@AA# + +  + +  + + + +BA +  ( "4 4  S ?    4  d g    4 4  4  4 4 4 4 n  X]" EE E E E T66)2STTTT U E  E  E EEEE:  )-!" ## #%## #  #  ####L d e e e6%%% .5  "& #',0")=,=, =,=, =,  =,  =,  =, =, =, =,=,!=,=, =, =, =, *!=,"#=,$ %=,& '=,( )=,* +=,=,=,  =,@ #QRRR ={333 .5b c"& #',0"%0t0t0t 0t0t 0t  0t  0t0t 0t 0t 0t0t!0t 0t0t 0t *!0t"#0t$ %0t& '0t0t0t  0tf'3I2JKK " %WmWm    D     WmWm Wm  S D  Q     Wm& 'WmWmWmLKWmty,L+MNN#xx  S Q    xx x xxxONx0x+J*KLL# = =' = =  = = = =ML =y,L+MNN# Q Q' Q Q  Q Q Q QON Qx+J*KLL# = =' = =  = = = =ML =y,L+MNN# Q Q' Q Q  Q Q Q QON Q bg" OO'O O O T66)2]^^^^ _ O  O  O OOOOB!=&QRRR (t444 .5LF"&"& #',0"+6`6`6` 6`6` 6`  6`  6` 6` 6`6` 6` 6` 6`6`!6`6` !6`"*#6`$%6`& '6`( )6`* +6`, -6`6`6`6`x "S#X "4S>D3H "TRUWZRZ^ " " " "\S4Z\S D0@\T\\\\ GKd3i$.ttTWY\T\~`dOd,DcCi)DI#6 GCLLU1XXc8A;>N5OY]bf    ) 3 # $d  a[ r8