O/jUUdZddlmZmZddlmZmZmZddlZddl Z ddl m Z ddl Z ddl Z ddlZddlZddlZddlZddlmZddlZddlmZdd lmZmZdd lmZmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%e jLe'Z(ejRgdfZ*gZ+ejXe*e-d<de*fdZ.GddZ/GddZ0de1fdZ2eGddZ3GddZ4y)z)Scheduling and job monitoring utilities. )contextmanager ExitStack) dataclassfieldasdictN)Path)SlurmJob)git_save) SlurmConfig SubmitRules)disable_logging setup_logging) DecoratedMain)try_load)XP_get_sigget_xp_preemption_callbackscallbackc.tj|yN)rappend)rs L/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/dora/shep.pyregister_preemption_callabackr(s  *c JeZdZ ddedej ededefdZdZ y) _SubmitItTarget main_pickledargvrequeue local_codecttjdtjtj tjdt jt5}|rtdj}|j|jdzz }|js+tjd|tjdt|j!t#j$}tjd||t'j(dd t+|g| ||j,z } | jstjd | tj.j1t jt j2| tj.j5d t+| tjd | tjdtj.ddlm} ||_| } t+| j<t j>d<|tj@ddtjdtCjD|} tjdtG| dddy#1swYyxYw)Nz4SubmitItTarget starting, python is %s, version is %rz CWD is %s.z.tarz2Could not find tar file %s to run the code locallyr zExtracting tar file %s to %starxf)cwdzCould not find code folder %srzSuccessfully changed dir to %szsys.path is now %rget_distrib_specRANKzLoading pickled mainzGoing into main)$rloggerinfosys executableversionosgetcwdrrresolveparentnameexistscriticalexit enter_contexttempfileTemporaryDirectoryr run_commandstrstempathremovechdirinsertdistribr)r!rankenvironr pickleloadsr) selfrr r!r"stack code_pathtar_path target_folderlocal_code_folderr)specmains r__call__z_SubmitItTarget.__call__-s JNNCKK 1 K- [ E I--/ $++y~~/FG(OO$XZbcHHQK $U%8%89T9T9V%W X  :HmT$$eT3x=%A}U$1HMM$A!(//1OO$CEVW ,*+3'8#9: <>OP 0#((; 1"DL#%D"%TYYBJJv CHHQRL KK. /<< -D KK) *   F=   s 0IKKc^ddlm}tD] }| |jst j d|j dk(r>t}|jjr|jjtjj|g|i|S)Nr r(r)rBr)rr!r-r7rCrrendezvous_filer5unlinksubmitithelpersDelayedSubmission)rGargskwargsr)rxps r checkpointz_SubmitItTarget.checkpointSs-- H J || HHQK   " "a 'B!!((*""))+11$HHHHrN)TF) __name__ __module__ __qualname__bytestpSequencer<boolrOrYrrrr,s:TX$)$U$"++c2B$T$!$L IrrceZdZdZdefdZedefdZedefdZ e gdfdZ e dd Z dd Z dd Zd edefd Zedej$efdZedZdZy)Sheepz[ A Sheep is a specific run for a given XP. Sheeps are managed by the Shepherd. rXcd||_d|_g|_g|_|jj rzt |j}t|trMt|dk(r|\|_|_yt|dk(r|\|_|_|_ytd||_yy)NzInvalid content for job file.) rXjob _other_jobs_dependent_jobs _job_filer5r isinstancetuplelen RuntimeError)rGrXcontents r__init__zSheep.__init__hs3779;= >> "t~~.G'5)w<1$18.DHd.\Q&GNDDHd.0D&'FGG" #rreturnc|jj|jjjjz Sr)rXfolderdorashepjob_filerGs rrjzSheep._job_filezs)ww~~ 1 1 : :::rc|jj|jjjjz Sr)rXrsrtru json_job_filerws r_json_job_filezSheep._json_job_file~s)ww~~ 1 1 ? ???rstandardc|y|jj|j|}|dk(r|rtd|Drd}|j dry|S)z1Return the current state of the `Sheep`. NUNKNOWNc3:K|]}|jdk7yw)r}N)state).0rgs r z#Sheep._get_state..s@c399 )@sMISSING CANCELLED)watcher get_statejob_idany startswith)rg other_jobsmoders r _get_statezSheep._get_states[ ; %%cjj$7 I *@Z@@"   K ( rcT|y|jj|j|S)DReturn True if the job is no longer running on the cluster. T)ris_doner)rgrs r_is_donezSheep._is_dones' ;{{""3::t44rcp|jy|jrdt|jdksJ|jg|jz}|D]*}|jj |j |r*yy|jjj |jj |S)rTr F)rgrirmrhrrr)rGrchainrgs rrz Sheep.is_dones 88    t''(A- --XXJ!5!55E !{{**3::t<  !88##++DHHOOTB Brcv|jr|jJt|jdksJ|jg|jz}d}|D]8}tj |g|}|dk(stj ||r6|cS|J|S|j |j|j|S)Nr COMPLETED)rirgrmrhrcrr)rGrrrrgs rrz Sheep.states   88' ''t''(A- --XXJ!5!55EE !((b$7K'u~~c4/H L !$ $$L??488T-=-=tD Drrc:|jj|dz S)Nz _0_log.out)rXrSrGrs r_logz Sheep._logswwVHJ"777rc|jy|jj}|jD]8}|j|jj s-|j}:|S)zPReturn the current job id, especially useful when using dependent jobs. N)rgrrirr5)rGrrgs rcurrent_job_idzSheep.current_job_ids\ 88 '' $Cyy$++- $ rcB|j}|y|j|S)z)Return the path to the main log. N)rrrs rlogz Sheep.logs&$$ >yy  rcd|jjd|jd}|j|d|jdz }|d|jj dz }|S)NzSheep(z, state=z, zsid=zargv=))rXsigrrgrr )rGouts r__repr__zSheep.__repr__shtww{{m8DJJL>< 88  T$--.b1 1C tww||nA&& rN)r{)rZr[r\__doc__rrppropertyrrjrz staticmethodrrrrr<rr^Optionalrrrrarrrcrccs#2#$;4;;@@@#%J 55 C E83848  C 0  !!rrcxcy)z7No logging logging function, passed to `Shepherd`. Nra)rs rno_logrs rcNeZdZUeed<eeZeje ed<y) _JobArray slurm_config)default_factorysheepsN) rZr[r\r __annotations__rlistrr^Listrcrarrrrs"48FBGGEN8rrcFeZdZdZefdedejegdffdZ dejede fdZ d edeje fd Zd edeje fd Zd ZedefdZde dedefdZdgdfdejej.dejej.deje fdZdZedefdZedefdZedefdZdej>efdZ dedededejBfdZ"dZ#edefd Z$d!e%fd"Z&y)#Shepherdz Takes care of the little jobs. Args: main (DecoratedMain): main function decorated by Dora. log (callable): log function, if provided should take a single string argument. rNrNc&||_|jjdd|jjdd|jjdd||_d|_d|_g|_g|_ |jy)NT)exist_okparentsF) rN_by_idmkdir_orphans_arraysr _in_job_array_existing_git_clone _to_cancel _to_submit_check_orphans)rGrNrs rrpzShepherd.__init__s  46 T48 D$7#(6: 68.0 rr rqcrt|trJ|jj|}t |S)z Given a list of of arguments, return the matching `Sheep`, which will contain both information on the `dora.xp.XP`, and on the latest job associated with that XP. )rkr<rNrrc)rGr rXs rget_sheep_from_argvzShepherd.get_sheep_from_argvs2 dC((( YY  d #RyrrcN|jj|}t|S)zj Returns a `Sheep` given the XP signature, if any exists, otherwise returns None. )rNget_xp_from_sigrc)rGrrXs rget_sheep_from_sigzShepherd.get_sheep_from_sigs" YY & &s +Ryrrc|j|z }|jr@|jj}|jj |}t |Sy)zu Returns the `Sheep` associated with the given `job_id`. If no sheep is found, returns None. N)r is_symlinkr2r4rNrrc)rGrlinkrrXs rget_sheep_from_job_idzShepherd.get_sheep_from_job_idsN {{V# ?? ,,.%%C**3/B9 rc@tjjy)zB Force an update of all job states with submitit. N)r rupdaterws rrzShepherd.updates !rrc#K|jrJ|jjt|d|_ dd|_y#d|_wxYww)z*Context manager to launch XP in job array.TNF)rrrr)rGrs r job_arrayzShepherd.job_array!sN%%%% y67! ' !&D D s:AA A AAsheeprulesc|j|j}|dk(r@|jrtj d|jj d|_n|dvrAtj d|jj d|j r\d|_nT|jrHtj d|jj d||j| d|_|jw|js$|jjt|||jd jk(sJ|jd jj|yy) z Decides whether to schedule a new job for the given sheep, based on the rules given in `rules`. Jobs are actually only scheduled once the `commit()` method is called. Nrz"Ignoring previously completed job )FAILEDr OUT_OF_MEMORYTIMEOUTr NODE_FAILz Previous job z failed or was canceledzCancelling previous job z with status )r)rgr replace_doner+debugrretryreplace cancel_lazyrrrrrr)rGrrrrs rmaybe_submit_lazyzShepherd.maybe_submit_lazy,s; 99 KKME #%%LL#EeiiFVFVEW!XY $EI(( }UYY-=-=,>>UVW;; $EI==LL#;EII?4??2#6#C#CC CC OOB  & & - -e 4 rrgdependent_jobsc|H|J|j7|xj|jgt|jzz c_yy|J|xj|gt|zz c_y)z Cancel a job. The job is actually cancelled only when `commit()` is called. You can either provide manually both a job and its dependents, or a sheep that will be automatically processed. N)rgrrri)rGrgrrs rrzShepherd.cancel_lazyIsn ;$ $$yy$EII;e6K6K1L#LL%= = OOutN';; ;Orc|jr"|j|jg|_d|_|jr:|jj d}|j ||jr9yy)zu Commit all changes registered so far with either `maybe_submit_lazy()` and `cancel_lazy()`. Nr)r_cancelrrpop_submit)rGrs rcommitzShepherd.commitYsZ ?? LL ) DO#' oo++A.I LL #oorc|jjj|jjjjz Sr)rNrtdirruby_idrws rrzShepherd._by_idgs/yy~~!!DIINN$7$7$=$===rc|jjj|jjjjz Sr)rNrtrruorphansrws rrzShepherd._orphansks/yy~~!!DIINN$7$7$?$???rc|jjj|jjjjz Sr)rNrtrruarraysrws rrzShepherd._arraysos/yy~~!!DIINN$7$7$>$>>>rjobscdg|Dcgc]}|jc}z}tjddj|t j |dycc}w)Nscancelz Running %s Tcheck)rr+rjoinsprun)rGrrg cancel_cmds rrzShepherd._cancelssH[$#?3CJJ#??  \388J#78 z&$@sAr4rscdtjd<t|j}t j ||j d|j d}|j}|dkDr|dzdk7r td|dz|d <d}n|}d |d <|dk(}|rd }|j} | r|j|z} | d |d <|sd ||d<|jr$d |d<|j3||jz|d<n ||d<|j|j|d<|j d} |j d} | | } | r<|jdg} | jdtjg| |d<i}|jd#dj!|j d|d<||d<|d=|d=|d=|d=|d=t"j%d||j&d |dd||S)!N1SLURM_KILL_BAD_EXITmax_num_timeoutpython)rsrrrz.Can only take <= 8 gpus, or multiple of 8 gpusnodesr GBmemzgpu:gresntasks_per_node cpus_per_taskcontainer_chdir force_chdir srun_argsz--container-workdirnodelist,additional_parametersgpus mem_per_gpu cpus_per_gpuone_task_per_node dependentszSlurm parameters %rT)job_namestderr_to_stdoutra)r0rDdict__dict__rS SlurmExecutorrr ValueErrorrrrrgetextendr1rr+rupdate_parameters)rGr4rsrrWexecutorr gpus_per_nodeno_gpusrrrrr additionals r_get_submitit_executorzShepherd._get_submitit_executorxsA,/ ()l++,))6::6G+H::h')   !8ax1} !QRR"aiF7OM MF7O!) M".. **]:C"e2JF5M#M?3F6N  ) )()F$ %))1*7,:S:S*S'(5F$ %))1*6*C*C' **%67jj/  ")O  ;3I   3RYY[A B"+F; (* ::j ! -%(XXfjj.D%EJz "*4&' 6N = ! > " & ' <  *F3""" !  rc 8|jjD]}|j}tj d|dt j ddtjd|dddgd d }|jjjjd Dcgc]}|s|jd d }}|r4tj d|dt j dg|zd |jycc}w)zCheck for orphaned jobs.zFound dirty tag z`, meaning a job might have been scheduled but Dora or Slurm crashed before the job id was saved.squeuez-uz-nz-oz%iz-hT)capture_outputr _rzFound orphan job ids z , will cancelrrN)riterdirr4r+warningrrr0getloginstdoutdecodestripsplitrR)rGdirtyr4proclineidss rrzShepherd._check_orphanss]]**, E::D NN-dV4TT U668T2;;=$dDRVW)-T;D26++2D2D2F2L2L2N2T2TUY2Zc$^b4::c?1%cCc!6se=IJ {S(5 LLN  ds .D6Dc#K|j|z }|j d|jy#|jwxYww)z,Context manager to enter a potential orphan.N)rtouchrR)rGr4tokens r _enter_orphanzShepherd._enter_orphans9 $    LLNELLNs A 8A A  A rc  |j}|j}|syt|dkD}|d}|jj |j |j j jtfd|DsJdd}|jr |rJdd}|r7tt|Dcgc]}|j jc}}n|j j}|r|jjdz|z} n|jjd z|z} |r|j| z } n|j j} | j!d |D]^}|j } |jj | | j"j%sE| j"j'`t)j*|j} |jj j,r sJd g} r0|j.$tj0|j|_|j3| 5t55}r<|j.J|j7tj8|j.|j;| | |}t55}|r|j7|j=|jD]B}r8|j.Jtj>|j |j.| |j j@||jj j,g}| jC|jDtGg||jst|jdk(sJtI|jD]d}||jdz k(}| d jJ}|jMd d|i| jC|jDtGg|fE dddg}|jr | dd}| dd} tO|| D];\}}|jJ| Dcgc]}|jJc}|Dcgc]}|jJc}tQ|d}|jRjUtWj*|t)jX|| |ft[|j\dt^jad|jJ||_1| |_2||_3|jh|jJz }|}|jk|j jljo|rg|j jl| jz }|j%r#|jo| jok(sJ|jk| |j jp}|j%r|j'|jk| |jjs|j } |jud|jJd|j jd| > ddddddycc}w#1swYxYwcc}wcc}w#1swY.xYw#1swYyxYw)Nr rc3dK|]'}|jjjk()ywr)rXrtr )rother use_git_saves rrz#Shepherd._submit..s$Ne588==))\9Ns-0z?All jobs inside an array must have the same value for git_save.Tz(Cannot use dependent jobs and job arraysF_array_r)rz(Cannot use local_code without git_save !r dependencyz afternotok:)r)r array_job_idsdependent_job_idsrwbzCreated job with id %szScheduled job z for sheep /);rrrmrNinit_xprXrtr allrrsortedrr4r _xp_submititrrQr5rRrEdumpsr"r get_new_cloner(rr8 enter_clonerbatch assign_cloner rsubmitrrangerrziprrz write_textjsondumpopenrjr+rrgrhrir symlink_torsr2_latest_submititget_namer)rGrrris_arrayfirstr!rname_sigr4submitit_folderrXrrrHr array_stack submitit_args dep_index last_job_idrrg other_jobjob_info_for_jsonr submitit_linklatestr,s @rrzShepherd._submits!! --  v;?q  %((#xx}}-- NvNN P O PN  " " K!K K<G &'I 'I JKHxx||H 99>>I-8D99>>C'(2D "llT1O#hh33Ot, ,EB II  b !!!((*""))+  , ||DII. 99>> $ $ K!K K<&( D44<'/'='=dii'HD $    %= Xy{= Xe//;;;##H$8$89Q9Q$RS224,WH \ --hnn.>?&--\E##77CCC --ehh8P8PQ%1588==%,diinn.G.G%IMKK0A RM RS#.."9#3#34999).|/F/F)G\I&/<3J3JQ3N&NG*.r(//K$667C{S^R_E`6a7c KK8I(ZM(Z[ \\ \$ N&&!%abBQx"&$/ X s!jjHL%M9i&6&6%MLZ)[y)*:*:)[$*<$8 %!$$// ;L0MN S$7eoot9TU 5szzB $(!(6%{{SZZ/ 7 7 9:&+XX__7K7K%KM$++-,446/:Q:Q:SSSS%00A22==?MMO!!/2yy))%((3>#**[aPTvVWA X;= X= X= X5(J> \ \6&N)[C= X= X= X= Xse9Z6  [(A[CZ;BZ;9A[;[ [[ 'G=[%[;[ [[ [[')'rZr[r\rrrr^Callabler<rpr_rcrrrrrrr rr rrSr rrrrrrrrrr rrr(rrrarrrrsMS ] cUD[1I  C(8Ucbkk%.@ C BKK4F " 'k''5u5K5P[5:AEEG04>>@$@@???'BGGH-' 9399-89=E=S=S9v #iXiXrr)5r contextlibrr dataclassesrrrr@loggingpathlibrrEr0 subprocessrr-r9typingr^rSr r confr r rrrrNrutilsrrXrrr getLoggerrZr+rRPreemptionCallbackrrrrrrcr<rrrrarrr^s000   */$$   8 $[[T*57rww127+,>+4I4Inttn c  99 9 NXNXr