Ë
    O/æjà'  ã                   ó¬   — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 ddl
mZmZ ddlmZmZ dZ G d„ d«      Z G d	„ d
«      Zdede	dedefd„Zy)é    N)ÚPathé   )ÚloggerÚutils)ÚDelayedSubmissionÚJobPathsÚSUBMITIT_PREEMPT_SIGNALc                   óš  ‡ — e Zd ZU dZej
                  j                  ed«      Zi Z	e
j                  e
j                  eef      ed<   de
j                  dd fˆ fd„Zdd„Zedefd	„«       Zedefd
„«       Zdefd„Zedefd„«       Zede
j4                  e   fd„«       Zedefd„«       Zedefd„«       Zede
j<                  e   fd„«       Zede
j<                  e   fd„«       Z ede!fd„«       Z"ede!fd„«       Z#ede!fd„«       Z$ede!fd„«       Z%ede!fd„«       Z&defd„Z'ede
j                  fd„«       Z(dede)ddfd„Z*de!ddfd„Z+ˆ xZ,S )ÚJobEnvironmentaV  Describe the environment inside which the job is running.
    This includes job id, number of GPUs available, ...

    This class can only be instantiated from a running submitit job.

    @plugin-dev: default implementation look for information into environment variables.
    Override _env to map environment variable to each property.
    ÚUSR2Ú_envÚargsÚreturnc                 ó`   •— | t         urt        ‰| �  | g|¢­Ž S ddlm} |j                  «       S )Nr   )Úplugins)r   ÚsuperÚ__new__Ú r   Úget_job_environment)Úclsr   r   Ú	__class__s      €ú`/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/submitit/core/job_environment.pyr   zJobEnvironment.__new__&   s1   ø€ Ø”nÑ$Ü‘7‘? 3Ð.¨Ò.Ð.åà×*Ñ*Ó,Ð,ó    Nc                 ó.   — | j                  «       | _        y ©N)ÚnameÚcluster©Úselfs    r   Ú__init__zJobEnvironment.__init__.   s   € Ø—y‘y“{ˆ�r   c                 óz   — | j                   }|j                  d«      r|d t        d«        }|j                  «       S )Nr   )Ú__name__ÚendswithÚlenÚlower)r   Úns     r   r   zJobEnvironment.name1   s:   € à�L‰LˆØ�:‰:Ð&Ô'ØÐ*”SÐ)Ó*Ð*Ð+ˆAØ�w‰w‹yÐr   c                 ól   — t         j                  d   }t        || j                  | j                  ¬«      S )zlProvides the paths used by submitit, including
        stdout, stderr, submitted_pickle and folder.
        ÚSUBMITIT_FOLDER)Újob_idÚtask_id)ÚosÚenvironr   r)   Úglobal_rank)r   Úfolders     r   ÚpathszJobEnvironment.paths8   s,   € ô
 —‘Ð-Ñ.ˆÜ˜ t§{¡{¸D×<LÑ<LÔMÐMr   c                 ód   — t         j                  j                  dd«      | j                  «       k(  S )z¥Tests if we are running inside this environment.

        @plugin-dev: assumes that the SUBMITIT_EXECUTOR variable has been
        set to the executor name
        ÚSUBMITIT_EXECUTORr   )r+   r,   Úgetr   r   s    r   Ú	activatedzJobEnvironment.activated@   s%   € ô �z‰z�~‰~Ð1°2Ó6¸$¿)¹)»+ÑEÐEr   c                 ó*   — t        j                  «       S r   )ÚsocketÚgethostnamer   s    r   ÚhostnamezJobEnvironment.hostnameH   s   € ä×!Ñ!Ó#Ð#r   c                 ó   — | j                   gS r   )r7   r   s    r   Ú	hostnameszJobEnvironment.hostnamesL   s   € à—‘ˆÐr   c                 óh   — | j                   r| j                   › d| j                  › �S | j                  S )NÚ_)Úarray_job_idÚarray_task_idÚ
raw_job_idr   s    r   r)   zJobEnvironment.job_idP   s4   € à×ÒØ×'Ñ'Ð(¨¨$×*<Ñ*<Ð)=Ð>Ð>à—?‘?Ð"r   c                 óB   — t         j                  | j                  d      S )Nr)   )r+   r,   r   r   s    r   r>   zJobEnvironment.raw_job_idW   s   € ä�z‰z˜$Ÿ)™) HÑ-Ñ.Ð.r   c                 ó€   — d}|| j                   vrd S t        j                  j                  | j                   |   d «      S )Nr<   ©r   r+   r,   r2   ©r   r&   s     r   r<   zJobEnvironment.array_job_id[   s4   € àˆØ §	¡	Ñ)ˆtÐQ¬r¯z©z¯~©~¸d¿i¹iÈ¹lÈDÓ/QÐQr   c                 ó€   — d}|| j                   vrd S t        j                  j                  | j                   |   d «      S )Nr=   rA   rB   s     r   r=   zJobEnvironment.array_task_id`   s4   € àˆØ §	¡	Ñ)ˆtÐQ¬r¯z©z¯~©~¸d¿i¹iÈ¹lÈDÓ/QÐQr   c                 ón   — t        t        j                  j                  | j                  d   d«      «      S )z!Total number of tasks for the jobÚ	num_tasksr   ©Úintr+   r,   r2   r   r   s    r   rE   zJobEnvironment.num_taskse   ó'   € ô ”2—:‘:—>‘> $§)¡)¨KÑ"8¸!Ó<Ó=Ð=r   c                 ón   — t        t        j                  j                  | j                  d   d«      «      S )z!Total number of nodes for the jobÚ	num_nodesr   rF   r   s    r   rJ   zJobEnvironment.num_nodesj   rH   r   c                 ón   — t        t        j                  j                  | j                  d   d«      «      S )zId of the current nodeÚnoder   rF   r   s    r   rL   zJobEnvironment.nodeo   s'   € ô ”2—:‘:—>‘> $§)¡)¨FÑ"3°QÓ7Ó8Ð8r   c                 ón   — t        t        j                  j                  | j                  d   d«      «      S )zGlobal rank of the taskr-   r   rF   r   s    r   r-   zJobEnvironment.global_rankt   s'   € ô ”2—:‘:—>‘> $§)¡)¨MÑ":¸AÓ>Ó?Ð?r   c                 ón   — t        t        j                  j                  | j                  d   d«      «      S )z/Local rank of the task, ie on the current node.Ú
local_rankr   rF   r   s    r   rO   zJobEnvironment.local_ranky   s'   € ô ”2—:‘:—>‘> $§)¡)¨LÑ"9¸1Ó=Ó>Ð>r   c                 óP  — dD �cg c]  }|› dt        | |«      › �‘Œ }}d}| j                  | j                  z  | j                  | j                  g}|t        ||«      D ��cg c]  \  }}|› dt        | |«      › d|› d�‘Œ c}}z  }dj	                  |«      }d|› d�S c c}w c c}}w )N)r)   r7   ú=)rO   rL   r-   ú(ú)z, zJobEnvironment()ÚgetattrrE   rJ   ÚzipÚjoin)r   r&   ÚinfoÚnamesÚtotalsÚtÚinfo_strs          r   Ú__repr__zJobEnvironment.__repr__~   s´   € ð 4JÖJ¨a�1�#�Q”w˜t QÓ'Ð(Ò)ÐJˆÐJØ5ˆØ—.‘. D§N¡NÑ2°D·N±NÀDÇNÁNÐSˆØ¼CÀÀvÓ<N×O±D°A°q�A�3�aœ  aÓ(Ð)¨¨1¨#¨QÒ/ÓOÑOˆØ—9‘9˜T“?ˆØ   
¨!Ð,Ð,ùò Kùó Ps   …BÁ!B"c                 óv   — d| j                   z   }t        t        |d «      }|€t        d|› dt        › d�«      ‚|S )NÚSIGzUnknown signal z*, you may need to unset or update env var z (Eg: USR2))ÚUSR_SIGrT   ÚsignalÚRuntimeErrorÚ_PREEMPT_SIG_ENV)r   r   Úouts      r   Ú_usr_sigzJobEnvironment._usr_sigˆ   sM   € à�s—{‘{Ñ"ˆÜ”f˜d DÓ)ˆØˆ;ÜØ! $ Ð'QÔRbÐQcÐcnÐoóð ð ˆ
r   r/   Ú
submissionc                 ó2  — t        | ||«      }t        j                  t        j                  |j                  «       t        j                  t        j                  |j                  «       t        j                  | j                  «       |j                  «       y)zÜSet up signals handler for the current executable.

        The default implementation checkpoint the given submission and requeues it.
        @plugin-dev: Should be adapted to the signals used in this cluster.
        N)ÚSignalHandlerr`   ÚSIGTERMÚbypassÚSIGCONTrd   Úcheckpoint_and_try_requeue)r   r/   re   Úhandlers       r   Ú_handle_signalszJobEnvironment._handle_signals’   sZ   € ô    e¨ZÓ8ˆô 	�‰”f—n‘n g§n¡nÔ5Ü�‰”f—n‘n g§n¡nÔ5ä�‰�d—m‘m“o w×'IÑ'IÕJr   Ú	countdownc                  ó   — y)zªRequeue the current job.

        @plugin-dev:Must be overridden by JobEnvironment implementations.
            Use self.job_id to find what need to be requeued.
        N© )r   rn   s     r   Ú_requeuezJobEnvironment._requeue¡   s   � r   ©r   N)-r"   Ú
__module__Ú__qualname__Ú__doc__r+   r,   r2   rb   r_   r   ÚtpÚClassVarÚDictÚstrÚ__annotations__ÚAnyr   r    Úclassmethodr   Úpropertyr   r/   Úboolr3   r7   ÚSequencer9   r)   r>   ÚOptionalr<   r=   rG   rE   rJ   rL   r-   rO   r\   rd   r   rm   rq   Ú__classcell__)r   s   @r   r   r      sN  ø… ñð �j‰j�n‰nÐ-¨vÓ6€GØ+-€Dˆ"�+‰+�b—g‘g˜c 3˜hÑ'Ñ
(Ó-ð-˜BŸF™Fð -Ð'7õ -ó#ð ð�Sò ó ðð ðN�xò Nó ðNðF˜4ó Fð ð$˜#ò $ó ð$ð ð˜2Ÿ;™; sÑ+ò ó ðð ð#˜ò #ó ð#ð ð/˜Cò /ó ð/ð ðR˜bŸk™k¨#Ñ.ò Ró ðRð ðR˜rŸ{™{¨3Ñ/ò Ró ðRð ð>˜3ò >ó ð>ð ð>˜3ò >ó ð>ð ð9�cò 9ó ð9ð ð@˜Sò @ó ð@ð ð?˜Cò ?ó ð?ð-˜#ó -ð ð˜Ÿ™ò ó ððK Xð KÐ;Lð KÐQUó Kð #ð ¨$÷ r   r   c                   ó  — e Zd Zdedededdfd„Zdefd„Zdde	d	e
j                  ej                     ddfd
„Zdde	d	e
j                  ej                     ddfd„Zdde	d	e
j                  ej                     ddfd„Zdd„Zy)rg   ÚenvÚ	job_pathsÚdelayedr   Nc                 ó’   — || _         || _        || _        t        j                  «       | _        t        j                  «       | _        y r   )rƒ   Ú
_job_pathsÚ_delayedr   Ú
get_loggerÚ_loggerÚtimeÚ_start_time)r   rƒ   r„   r…   s       r   r    zSignalHandler.__init__ª   s5   € ØˆŒØ#ˆŒØˆŒÜ×(Ñ(Ó*ˆŒÜŸ9™9›;ˆÕr   c                 óX  — t        j                   «       | j                  z
  }| j                  j                  dz  }t	        |dz  |dz
  «      }||k\  }|r,| j
                  j                  d|dz  d›d|dz  d›d�«       |S | j
                  j                  d|dz  d›d|dz  d›d�«       |S )	Né<   gš™™™™™é?iX  zJob has timed out. Ran z.0fz minutes out of requested z	 minutes.zJob has not timed out. Ran )r‹   rŒ   rˆ   Ú_timeout_minÚminrŠ   rW   )r   ÚwalltimeÚmax_walltimeÚguaranteed_walltimeÚ	timed_outs        r   Úhas_timed_outzSignalHandler.has_timed_out±   sÕ   € ô —9‘9“; ×!1Ñ!1Ñ1ˆØ—}‘}×1Ñ1°BÑ6ˆÜ! ,°Ñ"4°lÀWÑ6LÓMÐàÐ 3Ñ3ˆ	ÙØ�L‰L×ÑØ)¨(°R©-¸Ð)<Ð<VÐWcÐfhÑWhÐilÐVmÐmvÐwôð Ðð �L‰L×ÑØ-¨h¸©m¸CÐ-@Ð@ZÐ[gÐjlÑ[lÐmpÐZqÐqzÐ{ôð Ðr   ÚsignumÚframec                 óz   — | j                   j                  dt        j                  |«      j                  › �«       y )NzBypassing signal )rŠ   Úwarningr`   ÚSignalsr   )r   r–   r—   s      r   ri   zSignalHandler.bypassÄ   s,   € Ø�‰×ÑÐ0´·±ÀÓ1G×1LÑ1LÐ0MÐNÕOr   c           	      óÔ  — | j                  «       }|rdnd}| j                  j                  dt        j                  |«      j
                  › dt        j                  «       › d|› d�«       | j                  j                  }|dk7  r | j                  j                  d|› d	�«       y | j                  }|j                  |z
  }d
}t        |j                  d«      r"t        || j                   j"                  |«      }n|rd}|dk  rd}|r6d|› d�}	| j                  j                  |	«       t%        j&                  |	«      ‚| j                  j)                  |«       | j+                  «        y )Nz	timed-outÚ	preemptedúCaught signal ú on z: this job is ú.r   z;Not checkpointing nor requeuing since I am a slave (procid=ú).r   Ú
checkpointz timed-out and not checkpointableztimed-out too many timeszJob not requeued because: )r•   rŠ   r™   r`   rš   r   r5   r6   rƒ   r-   rW   rˆ   Ú_timeout_countdownÚhasattrÚfunctionÚ_checkpointr‡   Úsubmitted_pickler   ÚUncompletedJobErrorrq   Ú_exit)
r   r–   r—   r”   ÚcaseÚprocidr…   rn   Úno_requeue_reasonÚmessages
             r   rk   z(SignalHandler.checkpoint_and_try_requeueÈ   sM  € Ø×&Ñ&Ó(ˆ	Ù'‰{¨[ˆØ�‰×ÑØœVŸ^™^¨FÓ3×8Ñ8Ð9¸¼f×>PÑ>PÓ>RÐ=SÐSaÐbfÐagÐghÐiô	
ð —‘×%Ñ%ˆØ�QŠ;Ø�L‰L×ÑÐ [Ð\bÐ[cÐceÐfÔgàà—-‘-ˆØ×.Ñ.°Ñ:ˆ	ØÐÜ�7×#Ñ# \Ô2Ü +¨G°T·_±_×5UÑ5UÐW`Ó aÑÙØ BÐØ�qŠ=Ø :ÐÙð 3Ð3DÐ2EÀQÐGˆGØ�L‰L×Ñ˜gÔ&Ü×+Ñ+¨GÓ4Ð4à�‰×Ñ˜)Ô$Ø�
‰
�r   c                 óö  — | j                   j                  dt        j                  |«      j                  › dt        j                  «       › �«       | j                  j                  }|r | j                   j                  d|› d�«       y | j                  }t        |j                  d«      r?t        | j                  | j                  j                  | j                  j                  «       | j!                  «        y )Nr�   rž   z-Not checkpointing since I am a slave (procid=r    r¡   )rŠ   rW   r`   rš   r   r5   r6   rƒ   r-   rˆ   r£   r¤   r¥   r‡   r¦   r¢   r¨   )r   r–   r—   rª   r…   s        r   Úcheckpoint_and_exitz!SignalHandler.checkpoint_and_exité   s³   € à�‰×Ñ˜N¬6¯>©>¸&Ó+A×+FÑ+FÐ*GÀtÌF×L^ÑL^ÓL`ÐKaÐbÔcà—‘×%Ñ%ˆÙØ�L‰L×ÑÐ MÈfÈXÐUWÐXÔYàà—-‘-ˆÜ�7×#Ñ# \Ô2Ü˜Ÿ™ t§¡×'GÑ'GÈÏÉ×IiÑIiÔjØ�
‰
�r   c                 ód   — | j                   j                  d«       t        j                  d«       y )Nz,Exiting gracefully after preemption/timeout.éÿÿÿÿ)rŠ   rW   ÚsysÚexitr   s    r   r¨   zSignalHandler._exitø   s    € à�‰×ÑÐHÔIÜ�‰��r   r   rr   )r"   rs   rt   r   r   r   r    r~   r•   rG   rv   r€   ÚtypesÚ	FrameTyperi   rk   r®   r¨   rp   r   r   rg   rg   ©   s³   „ ð'˜Nð '°xð 'ÐJ[ð 'Ð`dó 'ð˜tó ñ&P˜Sð P¨¯©°U·_±_Ñ)Eð PÐQUó Pñ°ð ¸R¿[¹[ÈÏÉÑ=Yð Ðeió ñB¨#ð °b·k±kÀ%Ç/Á/Ñ6Rð Ð^bó ôr   rg   r…   Úfilepathrn   r   c                 ó  — t        j                  «       j                  d«       | j                  «       }|€y|j	                  | j
                  |«       t        j                  |«      5 }|j                  |«       ddd«       y# 1 sw Y   yxY w)zÚCall the checkpoint method and dump the updated delayed.

    Returns:
    --------
        no_requeue_reason: str
            a string explaining while there was no requeuing, else empty string if requeuing works
    zCalling checkpoint method.Nz!checkpoint function returned Noner   )	r   r‰   rW   Ú_checkpoint_functionÚset_timeoutr�   r   Útemporary_save_pathÚdump)r…   rµ   rn   Úckpt_delayedÚtmps        r   r¥   r¥   þ   s}   € ô ×ÑÓ×ÑÐ9Ô:Ø×/Ñ/Ó1€LØÐØ2Ø×Ñ˜W×1Ñ1°9Ô=Ü	×	"Ñ	" 8Ó	,ð °Ø×Ñ˜#Ô÷à÷àús   Á(BÂB)r+   r`   r5   r±   r‹   r³   Útypingrv   Úpathlibr   r   r   r   r   r   rb   r   rg   rG   ry   r¥   rp   r   r   ú<module>r¿      sg   ðó 
Û Û Û 
Û Û Û Ý ç ß .à,Ð ÷Pñ P÷fRñ RðjÐ*ð °dð Àsð Èsô r   