O/j~9UddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z m Z ddlmZdZiZej$edfed<Gd d e j*eZGd d e j.ZGd de j2Z dde dededededededej<eddfdZGddZ y)N)Path)corejob_environmentloggerutils)Rsubprocess.Popen['bytes'] _PROCESSESc JeZdZ ddejeefdedejedejdddf fd Z dd e de fd Z e defd Zdd edejeeffd Zdde ddfdZddZddZdej(fdZdej(ddfdZxZS)LocalJobNfolderjob_idtasksprocessr returnct||||d|_|j|_||_|jD] }||_ y)NF)super__init___cancel_at_deletion _sub_jobs_process)selfrrrrsjob __class__s W/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/submitit/local/local.pyrzLocalJob.__init__sJ /#( 59^^ NN $D#DM $ force_checkc2|jd}|dk7S)z#Override to avoid using the watcherjobStateRUNNING)get_info)rrstates rdonez LocalJob.done.s  + !!rcb |jjddS#t$rYywxYw)zState of the jobr!unknownUNKNOWN)r#get Exceptionrs rr$zLocalJob.state3s2 ==?&&z9= =  s " ..modec|j,d}|jjjrd}d|iS|jj }|d}d|iS|dkrd}d|iSd}d|iS)z,Returns information about the job as a dict.zNO PROCESS AND NO RESULTFINISHEDr!r"r INTERRUPTED)rpaths result_pickleexistspoll)rr,r$r3s rr#zLocalJob.get_info<s == .Ezz''..0"& &}}!!# <E E"" AX!EE""EE""rcheckcp|j*|jjtjyyN)r send_signalsignalSIGINT)rr4s rcancelzLocalJob.cancelLs' == $ MM % %fmm 4 %rcx|j.|jjtjyy)zBSends preemption / timeout signal to the job (for testing purpose)N)rr7LocalJobEnvironment_usr_sigr+s r _interruptzLocalJob._interruptPs- == $ MM % %&9&B&B&D E %rc|jr4|jjddk(s|jd|jj j r!tj|jdyy)Nr!r.F)r4) rr#r)r:r0r1r2r poprr+s r__del__zLocalJob.__del__Us_  # #==?&&z2j@ % ( :: # # * * , NN4;; - -rct|j}d|d<|j|jt|j<|S)Nr)dict__dict__rr r)routs r __getstate__zLocalJob.__getstate___s:4==!J == $&*mmJt{{ # rr$c|jj|tj|jd|_yr6)rDupdater r)rr)rr$s r __setstate__zLocalJob.__setstate__fs* U#"t{{D9 r))rN)F)force)TrN)__name__ __module__ __qualname__tpUnionrstrSequenceintOptionalrboolr%propertyr$Dictr#r:r>rAAnyrFrI __classcell__rs@rrrs #'<@ $s#$${{3 $ 89 $  $ """ s#S#rwwsCx/@# 5D5D5F .bff:"&&:T:rrc.eZdZdddddddZded d fd Zy ) r<SUBMITIT_LOCAL_JOB_IDSUBMITIT_LOCAL_NTASKSSUBMITIT_LOCAL_JOB_NUM_NODESSUBMITIT_LOCAL_NODEIDSUBMITIT_LOCAL_GLOBALIDSUBMITIT_LOCAL_LOCALID)r num_tasks num_nodesnode global_rank local_rank countdownrNc|j}tjjd|d|dt j t y)Nz Requeued job z (z remaining timeouts))rr get_loggerinfosysexitLOCAL_REQUEUE_RETURN_CODE)rrgjids r_requeuezLocalJobEnvironment._requeuews?kk  =R {BV!WX *+r)rLrMrN_envrSrorrr<r<ms.),3'0.  D,#,$,rr<c eZdZdZeZ ddejee fde de dejeddf fd Z edejefd Zd ej"ddffd Zd edeefd ZedefdZde fdZd ededefdZedejeefdefdZde dej8efdZxZS) LocalExecutoraLocal job executor This class is used to hold the parameters to run a job locally. In practice, it will create a bash file in the specified directory for each job, and pickle the task function and parameters. At completion, the job will also pickle the output. Logs are also dumped in the same directory. The submission file spawn several processes (one per task), with a timeout. Parameters ---------- folder: Path/str folder for storing job submission/output and logs. Note ---- - be aware that the log/output folder will be full of logs and pickled objects very fast, it may need cleaning. - use update_parameters to specify custom parameters (n_gpus etc...). Nrmax_num_timeoutmax_pickle_size_gbpythonrct|||||#tjtj n||_tjj|j}|jddy)N)rurtT)parentsexist_ok) rrshlexquoterk executablervrJobPathsget_first_id_independent_folderrmkdir)rrrtrurv indep_folderrs rrzLocalExecutor.__init__sg  1+  6<^ekk#..1 ~~EEdkkR 4$7rcdhS)z4Parameters that can be set through update_parameterssetuprq)clss r_valid_parameterszLocalExecutor._valid_parameterss yrkwargsc |jdddkDr td|jdd}|jdd}t|tjstd|d t d |Dstd|d t |dkDr|t |kDrt|d |d t| di|y)aoUpdate the parameters of the Executor. Valid parameters are: - timeout_min (float) - gpus_per_node (int) - visible_gpus (Sequence[int]) - tasks_per_node (int) - nodes (int). Must be 1 if specified - signal_delay_s (int): signal (lately: USR2) delay before timeout Other parameters are ignored nodesrz0LocalExecutor can use only one node. Use nodes=1 gpus_per_node visible_gpusrqzProvided visible_gpus=z is not an instance of Sequence.c3<K|]}t|tywr6) isinstancerS).0xs r z.s.s!7Q#a&!7s timeout_ming@signal_delay_sstderr_to_stdoutFr)rrr cuda_devicesrrrr)rrrr) parametersr)rangestart_controllerrjoinrrQpidlist)rrntasksn_gpusrgpusrjobs r_submit_commandzLocalExecutor._submit_commands$$%5q9$$_a8**>2> , 2uV} Wf8M";;!!7$!77++M3???../?D!__001CUK//%%gr2  $;;s7;;'7PTUZ[aUbPc  rc dj|jdtjt |j gS)N z-u -m submitit.core._submit)rrvrzr{rQrr+s r_submitit_command_strz#LocalExecutor._submitit_command_strs3xx&CU[[QTUYU`U`QaEbcddrcHd}|jjdd}||zS)Nrr)rr))rrrs r _num_taskszLocalExecutor._num_taskss)"oo112BAF~%%ruidcy)Nrq)rrrs r_make_submission_file_textz(LocalExecutor._make_submission_file_textsrstringcy)N0rq)rs r#_get_job_id_from_submission_commandz1LocalExecutor._get_job_id_from_submission_commandsrsubmission_file_pathcgSr6rq)rrs r_make_submission_commandz&LocalExecutor._make_submission_commands r)?N)rLrMrN__doc__r job_classrOrPrQrrSfloatrTr classmethodSetrrXrr rrVrrr staticmethodbytesrListrrYrZs@rrsrs}sD*I !$'#' 8d#88" 8  C 8  8""&&+6BFF6t66sx{,esee&C& #CCBHHUCZ4HSTbggclrrsrrrrrrrrrc ttj}|jt ||t t d|zt t |dd|rdndd||rdnd t jddt |g} t|} | r1d jt|tj| gz} tj| | | } | S) zTStarts a job controller, which is expected to survive the end of the python session.<r1rlocal) r]SUBMITIT_LOCAL_COMMANDSUBMITIT_LOCAL_TIMEOUT_SSUBMITIT_LOCAL_SIGNAL_DELAY_Sr_r^SUBMITIT_STDERR_TO_STDOUTSUBMITIT_EXECUTORCUDA_VISIBLE_DEVICESSUBMITIT_LOCAL_WITH_SHELLz-mzsubmitit.local._localz && )shellenv)rCosenvironrHrQrSrkr|rUrrrz subprocessPopen) rrrrrrrrrproc_cmd need_shellrs rrrs rzz CJJ!.1&!$Sk)9%:!;&)#n*=&>!%()9#r!)).#B .Es6{SHeJ;;tE{ejj.B-CCDxzsCG NrceZdZdZdefdZdejdejddfdZ dd Z dd Z dd e dejeje fd Zdd e ddfdZy) ControllerzThis controls a job: - instantiate each of the tasks - sends timeout signal - stops all tasks if one of them finishes - cleans up the tasks/closes log files when deleted rcttjd|_t j tjd|_ttjd|_ttjd|_ttjd|_ g|_ g|_ g|_ ttjd}t|rtjntj |_t%||_t)j(t(j*|j,y)Nr]rrrrr)rSrrrrzsplitr timeout_srrUrrstdoutsstderrsrQgetppidgetpidrrrr8SIGTERM_forward_signal)rr with_shells rrzController.__init__s"**%<=> {{2::.F#GH RZZ(BCD!"**-L"MN $RZZ0K%L M02 /1 /1 "**%@AB zrzz|ryy{C6l  fnnd&:&:;rsignumargsrNch|jD]} |j|y#t$rY#wxYwr6)rr7r*)rrrtasks rrzController._forward_signal-s<JJ D   (   s % 11c N|jjdt|jDcgc]-}t j |j|j |/}}|Dcgc]}|jjdc}|_ |jr |jn(|Dcgc]}|jjdc}|_ t|jD]}ttj}|j!t#|t#||j |j$j't)j*|j,d||j||j|dycc}wcc}wcc}w)NT)rya)rar`r\Fzutf-8)rrstderrstdoutencoding)rrrrrr}rropenrrrrrCrrrHrQrappendrrr)rrr0prs r start_taskszController.start_tasks4s8 4(CHCUVa TXXq9VV49:q c*: '+'<'.Ns+QAFFH,<+QgMbP?c3@K|]}|jduywr6rrs rrz(Controller.kill_tasks..Ps>2 $C   %fmm#+Qdjj+Q(Q 5!<<<JJsO  $  t||+%'" dl A GGI rfreqc|jsJd||jz}||j|jz z}t|D]}|jDcgc]}|j }}t d|Dr|cS||k(r#|j tjtjd|z |jDcgc]}|j c}Scc}wcc}w)aWaits for all tasks to finish or to time-out. Returns ------- Sequence[Optional[int]]: Exit codes of each task. Some tasks might still have not exited, but they will have received the "timed-out" signal. zNothing to do!c3$K|]}|du ywr6rqres rrz"Controller.wait..hs5Q1D=5sr) rrrrr3rrr<r=rr)rrtimeoutalmost_timeoutstepr exit_codess rwaitzController.waitXszz+++z'$2E2E!EF'N #D,0JJ7q!&&(7J75*55!!~%$$%8%A%A%CD JJsTz " ##'**-Q--8.s C4C9 max_retryct|D][} |j|j}td|D}|s |j y |j ]y#|j wxYw)Nc3.K|] }|tk(ywr6)rmrs rrz!Controller.run..xsQa#<<Qs)rrr rr)rr _r requeues rrunzController.runqssy! "A "  "!YY[ QjQQ!! "!s 4A++A=rK)))rLrMrNrrrr8SignalsrOrXrrrrSrRrTr rrqrrrrsw rs  77  9; BGGC44 5:M:txx{M:` ,/88 , rD))rp"         ;;s !Dh"h"r