Ë
    P/æjÕ,  ã                   óä  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z
 d dlmZmZmZmZ  ej                   d«      Z ej$                  «       Z ej(                  d«      Zej-                  e«       ej/                  e«       ej1                  ej2                  «       de_         G d„ d	e«      Zdd
ee   defd„Zdd
ee   defd„Z dee   defd„Z! G d„ d«      Z"	 ddee"   dedefd„Z#y)é    N)ÚLock)Údefault_timer)ÚAnyÚListÚOptionalÚSequenceÚstrobelight_function_profilerzB%(name)s, line %(lineno)d, %(asctime)s, %(levelname)s: %(message)sFc                   ó   — e Zd ZdZy)ÚStrobelightCLIProfilerErrorzC
    Raised when an error happens during strobelight profiling
    N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© ó    úk/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/torch/_strobelight/cli_function_profiler.pyr   r      s   „ òr   r   ÚpidÚreturnc                 ó~   — d}| xs t        j                  «       } t        j                  |j                  | «      «      S )zFReturns the link to the process's namespace, example: pid:[4026531836]z/proc/{}/ns/pid)ÚosÚgetpidÚreadlinkÚformat)r   ÚPID_NAMESPACE_PATHs     r   Ú_pid_namespace_linkr   !   s3   € à*ÐØ
Ò
”—‘“€CÜ�;‰;Ð)×0Ñ0°Ó5Ó6Ð6r   c                 óˆ   — | xs t        j                  «       } t        | «      }t        ||j	                  d«      dz   d «      S )z"Returns the process's namespace idú[é   éÿÿÿÿ)r   r   r   ÚintÚfind)r   Úlinks     r   Ú_pid_namespacer#   (   s;   € à
Ò
”—‘“€CÜ˜sÓ#€DÜˆt�D—I‘I˜c“N QÑ&¨Ð,Ó-Ð-r   Úcommandc                 ó$   — dj                  | «      S )Nú )Újoin)r$   s    r   Ú_command_to_stringr(   /   s   € Ø�8‰8�GÓÐr   c                   óÜ   — e Zd ZdZ e«       Zddddddddddd	œ
d
ededede	dedede
ee	      de
ee	      dedefd„Zd"d„Zd#deddfd„Zd"d„Zd"d„Zdeddfd„Zdefd„Zdeded edefd!„Zy)$ÚStrobelightCLIFunctionProfileraŸ  
    Note: this is a Meta only tool.

    StrobelightCLIFunctionProfiler can be used to profile a python function and
    generate a strobelight link with the results. It works on meta servers but
    does not requries an fbcode target.
    When stop_at_error is false(default), error during profiling does not prevent
    the work function from running.

    Check function_profiler_example.py for an example.
    FiX  g    ÐcAzpytorch-strobelight-ondemandé<   Né   )
Ústop_at_errorÚmax_profile_duration_secÚsample_eachÚrun_user_nameÚtimeout_wait_for_running_secÚtimeout_wait_for_finished_secÚrecorded_env_variablesÚsample_tagsÚstack_max_lenÚasync_stack_max_lenr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   c       
         ó‚   — || _         || _        || _        || _        || _        || _        d | _        d | _        || _        y ©N)	r-   r.   r/   r0   r1   r2   Úcurrent_run_idÚprofile_resultr4   )Úselfr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   s              r   Ú__init__z'StrobelightCLIFunctionProfiler.__init__C   sN   € ð +ˆÔØ(@ˆÔ%Ø&ˆÔØ*ˆÔØ,HˆÔ)Ø-JˆÔ*ð .2ˆÔØ37ˆÔØ&ˆÕr   r   c                 óà  — t        j                  «       }t        |«      }ddddddddt        | j                  «      › d	t        | j
                  d
z  «      › d|› d|› �g}| j                  r;|j                  d«       |j                  dj                  | j                  «      «       t        j                  dt        |«      «       t        j                  |d¬«      }|j                  j                  d«      }t        j                  d|«       |j                   dk7  rt#        d|› �«      ‚t%        j&                  d|«      x}r t        |j)                  d«      «      | _        y t#        d|› �«      ‚)NÚstrobeclientÚrunz
--profilerÚpyperfz--eventÚcyclesz--asyncz--sample-intervalz--duration-msiè  z--pidú:z--sample-tagsú,úrunning command: %sT©Úcapture_outputúutf-8úoutput:
{%s}r   z:failed to start strobelight profiling, error in run_async:zINFO Run Id: (-?\d+)r   z9failed to start strobelight profiling, unexpected result )r   r   r#   r    r/   r.   r4   Úappendr'   ÚloggerÚdebugr(   Ú
subprocessr?   ÚstderrÚdecodeÚ
returncoder   ÚreÚsearchÚgroupr9   )r;   Ú	processIdÚ	namespacer$   ÚresultÚoutputÚmatchs          r   Ú
_run_asyncz)StrobelightCLIFunctionProfiler._run_async]   sW  € Ü—I‘I“Kˆ	Ü" 9Ó-ˆ	àØØØØØØØÜ�4×#Ñ#Ó$Ð%ØÜ�4×0Ñ0°4Ñ7Ó8Ð9ØØˆk˜˜9˜+Ð&ð
ˆð  ×ÒØ�N‰N˜?Ô+Ø�N‰N˜3Ÿ8™8 D×$4Ñ$4Ó5Ô6ä�‰Ð*Ô,>¸wÓ,GÔHÜ—‘ ¸Ô=ˆØ—‘×%Ñ% gÓ.ˆÜ�‰�_ fÔ-à×Ñ Ò!Ü-ØLÈVÈHÐUóð ô —I‘IÐ5°vÓ>Ð>ˆ5Ð>Ü"% e§k¡k°!£nÓ"5ˆDÔØä)ØGÈÀxÐPó
ð 	
r   Úcounterc                 óD  — |dkD  rt        d«      ‚ddd| j                  › g}t        j                  dt	        |«      «       t        j                  |d¬«      }|j                  j                  d	«      }t        j                  d
|«       |j                  dk7  rt        d|› �«      ‚t        j                  d|«      x}rU|j                  d«      }|dk(  ry |dk(  r*t        j                  d«       | j                  |dz   «       y t        d|› d�«      ‚t        d|› d�«      ‚)Né   z*wait_for_running called more than 20 timesr>   ÚgetRunStatusú--run-idrD   TrE   rG   rH   r   zAfailed to start strobelight profiling, error in wait_for_running:zProfile run status: (.*)r   ÚRUNNINGÚ	PREPARINGé
   zunexpected z phaseúunexpected output
: r&   )r   r9   rJ   rK   r(   rL   r?   rM   rN   rO   rP   rQ   rR   ÚtimeÚsleepÚ_wait_for_running)r;   rY   r$   rU   rV   rW   Úcurrent_statuss          r   rd   z0StrobelightCLIFunctionProfiler._wait_for_running†   s#  € Ø�RŠ<Ü-Ø<óð ð " >°:À$×BUÑBUÐAVÐXˆÜ�‰Ð*Ô,>¸wÓ,GÔHÜ—‘ ¸Ô=ˆØ—‘×%Ñ% gÓ.ˆÜ�‰�_ fÔ-à×Ñ Ò!Ü-ØSÐTZÐS[Ð\óð ô —I‘IÐ8¸&ÓAÐAˆ5ÐAØ"Ÿ[™[¨›^ˆNØ Ò*ØØ ;Ò.Ü—
‘
˜2”Ø×&Ñ& w°¡{Ô3Øä1°KÀÐ?OÈvÐ2VÓWÐWä)Ð,AÀ&ÀÈÐ*KÓLÐLr   c                 óî  — dddt        | j                  «      g}t        j                  dt	        |«      «       t        j                  |d¬«      }|j                  j                  d«      }t        j                  d|«       |j                  d	k7  rt        d
|› �«      ‚t        j                  d|«      x}r2|j                  d«      }|j                  d«      ry t        d|› d�«      ‚t        d|› d�«      ‚)Nr>   ÚstopRunr]   rD   TrE   rG   rH   r   z<failed to stop strobelight profiling, return code is not 0 :úINFO ::1:(.*)r   zSuccess!z*failed to stop strobelight profiling, got z resultra   r&   )Ústrr9   rJ   rK   r(   rL   r?   rM   rN   rO   r   rP   rQ   rR   Ú__contains__)r;   r$   rU   rV   rW   re   s         r   Ú	_stop_runz(StrobelightCLIFunctionProfiler._stop_run¤   sï   € Ø! 9¨j¼#¸d×>QÑ>QÓ:RÐSˆÜ�‰Ð*Ô,>¸wÓ,GÔHÜ—‘ ¸Ô=ˆØ—‘×%Ñ% gÓ.ˆÜ�‰�_ fÔ-à×Ñ Ò!Ü-ØNÈvÈhÐWóð ô —I‘I˜o¨vÓ6Ð6ˆ5Ð6Ø"Ÿ[™[¨›^ˆNØ×*Ñ*¨:Ô6Øä1Ø@ÀÐ@PÐPWÐXóð ô *Ð,AÀ&ÀÈÐ*KÓLÐLr   c                 óà  — dddt        | j                  «      g}t        j                  dt	        |«      «       t        j                  |d¬«      }|j                  j                  d«      }t        j                  d|«       |j                  d	k7  rt        d
|› �«      ‚t        j                  d|«      x}rg|j                  d«      }|j                  d«      r&t        j                   d«       | j#                  «        y |j                  d«      st        d|› �«      ‚g | _        t        j&                  d|«      D ]2  }| xj$                  |d	   z  c_        t        j)                  |d	   «       Œ4 y )Nr>   r\   r]   rD   TrE   rG   rH   r   z<failed to extract profiling results, return code is not 0 : rh   r   zProfile run status: PROCESSINGr`   z!Profile run finished with SUCCESSz9failed to extract profiling results, unexpected response zF(Total samples(.*)|GraphProfiler(.*)|Icicle view \(python stack\)(.*)))ri   r9   rJ   rK   r(   rL   r?   rM   rN   rO   r   rP   rQ   rR   rj   rb   rc   Ú_get_resultsr:   ÚfindallÚinfo)r;   r$   rU   rV   rW   re   Úitems          r   rm   z+StrobelightCLIFunctionProfiler._get_results»   sK  € Ø! >°:¼sÀ4×CVÑCVÓ?WÐXˆÜ�‰Ð*Ô,>¸wÓ,GÔHÜ—‘ ¸Ô=ˆØ—‘×%Ñ% gÓ.ˆÜ�‰�_ fÔ-à×Ñ Ò!Ü-ØNÈvÈhÐWóð ô —I‘I˜o¨vÓ6Ð6ˆ5Ð6Ø"Ÿ[™[¨›^ˆNØ×*Ñ*Ð+KÔLÜ—
‘
˜2”Ø×!Ñ!Ô#ØØ#×0Ñ0Ð1TÔUÜ1ØOÐPVÈxÐXóð ð !ˆÔÜ—J‘JØUØó
ò 	!ˆDð ×Ò 4¨¡7Ñ*ÕÜ�K‰K˜˜Q™Õ ñ	!r   Úcollect_resultsc                 óú   — 	 | j                  «        t        j                  d«       t        j                  d«       |sy | j	                  «        y # t
        $ r!}t        j                  dd¬«       Y d }~y d }~ww xY w)Nzstrobelight profiling stoppedzcollection stoppedzerror during stop_strobelightT©Úexc_info)rk   rJ   ro   rK   rm   Ú	ExceptionÚwarning)r;   rq   Úerrors      r   Ú_stop_strobelight_no_throwz9StrobelightCLIFunctionProfiler._stop_strobelight_no_throwÚ   sd   € ð	Kà�N‰NÔÜ�K‰KÐ7Ô8ä�L‰LÐ-Ô.á"Øà×ÑÕøÜò 	KÜ�N‰NÐ:ÀTˆN×JÑJûð	Kús   ‚<A ¿A Á	A:ÁA5Á5A:c                 ó:  — d}	 | j                  «        d}t        j                  d| j                  «       | j	                  «        t        j                  d«       y# t
        $ r5}t        j                  dd¬«       |r| j                  d¬«       Y d }~yd }~ww xY w)NFTzstrobelight run id is: %szstrobelight profiling runningzerror during start_strobelight:rs   ©rq   )rX   rJ   ro   r9   rd   ru   rv   rx   )r;   Ústrobelight_startedrw   s      r   Ú_start_strobelightz1StrobelightCLIFunctionProfiler._start_strobelightí   s†   € Ø#Ðð	Ø�O‰OÔØ"&ÐÜ�K‰KÐ3°T×5HÑ5HÔIØ×"Ñ"Ô$Ü�K‰KÐ7Ô8Øøäò 	Ü�N‰NÐ<ÀtˆNÔLÙ"Ø×/Ñ/ÀÐ/ÔFÜûð		ús   „AA Á	BÁ%+BÂBÚwork_functionÚargsÚkwargsc                 ó`  — d | _         d | _        t        j                  j	                  d«      x}�r$|s4| j
                  rt        d«      ‚t        j                  d«        ||i |¤ŽS | j                  «       }|s]| j
                  r)t        j                  j                  «        t        d«      ‚ ||i |¤Ž}t        j                  j                  «        |S 	 t        j                  d«       t        «       } ||i |¤Ž}t        «       }||z
  }	t        j                  d|	«       | j                  d¬«       t        j                  j                  «        |S y # t        $ rN}
t        j                  dd¬	«       | j                  d¬«       t        j                  j                  «        |
‚d }
~
ww xY w)
NFzconcurrent runs not supportedz%failed to start strobelight profilingzcollection startedzwork function took %s secondsTrz   zwork function throw exceptionrs   )r9   r:   r*   Ú_lockÚacquirer-   r   rJ   rv   r|   ÚreleaserK   Útimerro   rx   ru   )r;   r}   r~   r   ÚlockedÚstartedrU   ÚstartÚendÚ
total_timerw   s              r   Úprofilez&StrobelightCLIFunctionProfiler.profileý   s‡  € Ø"ˆÔØ"ˆÔä3×9Ñ9×AÑAÀ%ÓHÐHˆ6ÑHÙØ×%Ò%Ü5Ð6UÓVÐVä—‘Ð>Ô?Ù$ dÐ5¨fÑ5Ð5à×-Ñ-Ó/ˆGÙØ×%Ò%Ü2×8Ñ8×@Ñ@ÔBÜ5Ø?óð ñ '¨Ð7°Ñ7�Ü.×4Ñ4×<Ñ<Ô>Ø�ðÜ—‘Ð1Ô2Ü›�Ù&¨Ð7°Ñ7�Ü“g�Ø  5™[�
Ü—‘Ð;¸ZÔHØ×/Ñ/ÀÐ/ÔEÜ.×4Ñ4×<Ñ<Ô>Ø�ð9 Iøô: ò Ü—‘Ð>È�ÔNØ×/Ñ/ÀÐ/ÔFÜ.×4Ñ4×<Ñ<Ô>Ø�ûð	ús   ÃA=E Å	F-ÅA	F(Æ(F-)r   N)r   )r   r   r   r   r   r�   Úboolr    Úfloatri   r   r   r<   rX   rd   rk   rm   rx   r|   r   rŠ   r   r   r   r*   r*   3   s  „ ñ
ñ ‹F€Eð
 $Ø(/Ø Ø;Ø,.Ø-/Ø6:Ø+/Ø Ø#&ò'ð ð'ð #&ð	'ð
 ð'ð ð'ð '*ð'ð (+ð'ð !)¨¨c©Ñ 3ð'ð ˜d 3™iÑ(ð'ð ð'ð !ó'ó4'
ñRM¨ð M°Tó Mó<Mó.!ð>KàðKð 
óKð& Dó ð % Sð %°ð %Àð %Èô %r   r*   Úprofilerr   c                 óF   ‡ — ‰ st        di |¤ŽŠ dt        dt        fˆ fd„}|S )Nr}   r   c                 ól   •‡ — t        j                  ‰ «      dt        dt        dt        fˆˆ fd„«       }|S )Nr~   r   r   c                  ó0   •—  ‰j                   ‰g| ¢­i |¤ŽS r8   )rŠ   )r~   r   r�   r}   s     €€r   Úwrapper_functionz@strobelight.<locals>.strobelight_inner.<locals>.wrapper_function1  s    ø€ à#�8×#Ñ# MÐC°DÒC¸FÑCÐCr   )Ú	functoolsÚwrapsr   )r}   r‘   r�   s   ` €r   Ústrobelight_innerz&strobelight.<locals>.strobelight_inner0  s@   ù€ Ü	�‰˜Ó	'ð	D¤Cð 	D´3ð 	D¼3õ 	Dó 
(ð	Dð  Ðr   r   )r*   r   )r�   r   r”   s   `  r   Ústrobelightr•   *  s0   ø€ ñ Ü1Ñ;°FÑ;ˆð ¬ð  ´õ  ð Ðr   r8   )$r’   Úloggingr   rP   rL   rb   Ú	threadingr   Útimeitr   r„   Útypingr   r   r   r   Ú	getLoggerrJ   ÚStreamHandlerÚconsole_handlerÚ	FormatterÚ	formatterÚsetFormatterÚ
addHandlerÚsetLevelÚINFOÚ	propagateru   r   r    ri   r   r#   r(   r*   r•   r   r   r   ú<module>r¤      s  ðó Û Û 	Û 	Û Û Ý Ý )ß 0Ó 0ð 
ˆ×	Ñ	Ð:Ó	;€à'�'×'Ñ'Ó)€ØˆG×ÑØHó€	ð × Ñ ˜YÔ 'à × Ñ �/Ô "Ø ‡��—‘Ô Ø€Ô ô )ô ñ7˜X c™]ð 7°có 7ñ.˜ ™ð .°ó .ð ¨¡ð °#ó ÷oñ oðp :>ñØÐ5Ñ6ðØILðàôr   