
    VhM,                     6   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZmZmZ d dlmZ  ej"                  d      Z ej&                         Z ej*                  d      Zej/                  e       ej1                  e       ej3                  ej4                         de_         ed	      Z ed
      Z G d de      Zddee    de!fdZ"ddee    de fdZ#dee!   de!fdZ$ G d d      Z%	 ddee%   dedeeeef   geeee   f   f   fdZ&y)    N)Sequence)Lock)AnyCallableOptionalTypeVar)	ParamSpecstrobelight_function_profilerzB%(name)s, line %(lineno)d, %(asctime)s, %(levelname)s: %(message)sF_P_Rc                       e Zd ZdZy)StrobelightCLIProfilerErrorzC
    Raised when an error happens during strobelight profiling
    N)__name__
__module____qualname____doc__     ^/home/dcms/DCMS/lib/python3.12/site-packages/torch/utils/_strobelight/cli_function_profiler.pyr   r      s    r   r   pidreturnc                 ~    d}| xs t        j                         } t        j                  |j                  |             S )zFReturns the link to the process's namespace, example: pid:[4026531836]z/proc/{}/ns/pid)osgetpidreadlinkformat)r   PID_NAMESPACE_PATHs     r   _pid_namespace_linkr   %   s3    *

C;;)00566r   c                     | xs t        j                         } t        |       }t        ||j	                  d      dz   d       S )z"Returns the process's namespace id[   )r   r   r   intfind)r   links     r   _pid_namespacer&   ,   s;    

Cs#DtDIIcNQ&,--r   commandc                 $    dj                  |       S )N )join)r'   s    r   _command_to_stringr+   3   s    88Gr   c                      e Zd ZdZ e       Zddddddddddd	
d
ededede	dedede
ee	      de
ee	      dedefdZd"dZd#deddfdZd"dZd"dZdeddfdZdefdZdeeef   dej,                  d ej.                  de
e   fd!Zy)$StrobelightCLIFunctionProfilera  
    Note: this is a meta only tool.

    StrobelightCLIFunctionProfiler can be used to profile a python function and
    generate a strobelight link with the results. It works on meta servers but
    does not requries an fbcode target.
    When stop_at_error is false(default), error during profiling does not prevent
    the work function from running.

    Check function_profiler_example.py for an example.
    FiX  g    cAzpytorch-strobelight-ondemand<   N   )
stop_at_errormax_profile_duration_secsample_eachrun_user_nametimeout_wait_for_running_sectimeout_wait_for_finished_secrecorded_env_variablessample_tagsstack_max_lenasync_stack_max_lenr0   r1   r2   r3   r4   r5   r6   r7   r8   r9   c       
         t    || _         || _        || _        || _        || _        || _        d | _        || _        y N)r0   r1   r2   r3   r4   r5   current_run_idr7   )selfr0   r1   r2   r3   r4   r5   r6   r7   r8   r9   s              r   __init__z'StrobelightCLIFunctionProfiler.__init__G   sF     +(@%&*,H)-J* .2&r   r   c                    t        j                         }t        |      }ddddddddt        | j                         d	t        | j
                  d
z         d| d| g}| j                  r;|j                  d       |j                  dj                  | j                               t        j                  dt        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                   dk7  rt#        d|       t%        j&                  d|      x}r t        |j)                  d            | _        y t#        d|       )Nstrobeclientrunz
--profilerpyperfz--eventcyclesz--asyncz--sample-intervalz--duration-msi  z--pid:z--sample-tags,running command: %sTcapture_outpututf-8output:
{%s}r   z:failed to start strobelight profiling, error in run_async:zINFO Run Id: (-?\d+)r!   z9failed to start strobelight profiling, unexpected result )r   r   r&   r#   r2   r1   r7   appendr*   loggerdebugr+   
subprocessrA   stderrdecode
returncoder   researchgroupr<   )r=   	processId	namespacer'   resultoutputmatchs          r   
_run_asyncz)StrobelightCLIFunctionProfiler._run_async`   sW   IIK	"9-	4##$%4004789k9+&
  NN?+NN388D$4$456*,>w,GH=%%g._f-!-LVHU  II5v>>5>"%ekk!n"5D)GxP
 	
r   counterc                 D   |dkD  rt        d      ddd| j                   g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d	      }t        j                  d
|       |j                  dk7  rt        d|       t        j                  d|      x}rU|j                  d      }|dk(  ry |dk(  r*t        j                  d       | j                  |dz          y t        d| d      t        d| d      )N   z*wait_for_running called more than 20 timesr@   getRunStatus--run-idrF   TrG   rI   rJ   r   zAfailed to start strobelight profiling, error in wait_for_running:zProfile run status: (.*)r!   RUNNING	PREPARING
   zunexpected z phaseunexpected output
: r)   )r   r<   rL   rM   r+   rN   rA   rO   rP   rQ   rR   rS   rT   timesleep_wait_for_running)r=   r[   r'   rW   rX   rY   current_statuss          r   rf   z0StrobelightCLIFunctionProfiler._wait_for_running   s#   R<-<  ">:$BUBUAVX*,>w,GH=%%g._f-!-STZS[\  II8&AA5A"[[^N*;.

2&&w{31K?Ov2VWW),A&*KLLr   c                    dddt        | j                        g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                  d	k7  rt        d
|       t        j                  d|      x}r2|j                  d      }|j                  d      ry t        d| d      t        d| d      )Nr@   stopRunr_   rF   TrG   rI   rJ   r   z<failed to stop strobelight profiling, return code is not 0 :INFO ::1:(.*)r!   zSuccess!z*failed to stop strobelight profiling, got z resultrc   r)   )strr<   rL   rM   r+   rN   rA   rO   rP   rQ   r   rR   rS   rT   __contains__)r=   r'   rW   rX   rY   rg   s         r   	_stop_runz(StrobelightCLIFunctionProfiler._stop_run   s    !9j#d>Q>Q:RS*,>w,GH=%%g._f-!-NvhW  IIov6656"[[^N**:61@@PPWX  *,A&*KLLr   c                    dddt        | j                        g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                  d	k7  rt        d
|       t        j                  d|      x}rg|j                  d      }|j                  d      r&t        j                   d       | j#                          y |j                  d      st        d|       t        j$                  d|      D ]  }t        j'                  |d	           y )Nr@   r^   r_   rF   TrG   rI   rJ   r   z<failed to extract profiling results, return code is not 0 : rj   r!   zProfile run status: PROCESSINGrb   z!Profile run finished with SUCCESSz9failed to extract profiling results, unexpected response zF(Total samples(.*)|GraphProfiler(.*)|Icicle view \(python stack\)(.*)))rk   r<   rL   rM   r+   rN   rA   rO   rP   rQ   r   rR   rS   rT   rl   rd   re   _get_resultsfindallinfo)r=   r'   rW   rX   rY   rg   items          r   ro   z+StrobelightCLIFunctionProfiler._get_results   s.   !>:s4CVCV?WX*,>w,GH=%%g._f-!-NvhW  IIov6656"[[^N**+KL

2!!##001TU1OPVxX  JJU
 	!D KKQ 		!r   collect_resultsc                     	 | j                          t        j                  d       t        j                  d       |sy | j	                          y # t
        $ r t        j                  dd       Y y w xY w)Nzstrobelight profiling stoppedzcollection stoppedzerror during stop_strobelightTexc_info)rm   rL   rq   rM   ro   	Exceptionwarning)r=   rs   s     r   _stop_strobelight_no_throwz9StrobelightCLIFunctionProfiler._stop_strobelight_no_throw   s`    	KNNKK78LL-." 	KNN:TNJ	Ks   <A A  A32A3c                 ,   d}	 | j                          d}t        j                  d| j                         | j	                          t        j                  d       y# t
        $ r. t        j                  dd       |r| j                  d       Y yw xY w)NFTzstrobelight run id is: %szstrobelight profiling runningzerror during start_strobelight:ru   rs   )rZ   rL   rq   r<   rf   rw   rx   ry   )r=   strobelight_starteds     r   _start_strobelightz1StrobelightCLIFunctionProfiler._start_strobelight   s    #	OO"&KK3T5H5HI""$KK78 	NN<tNL"///F		s   AA 4BBwork_functionargskwargsc                    d | _         t        j                  j                  d      x}r|s4| j                  rt        d      t        j                  d        ||i |S | j                         }|s]| j                  r)t        j                  j                          t        d       ||i |}t        j                  j                          |S 	 t        j                  d        ||i |}| j                  d       t        j                  j                          |S y # t        $ rN}t        j                  dd       | j                  d       t        j                  j                          |d }~ww xY w)	NFzconcurrent runs not supportedz%failed to start strobelight profilingzcollection startedTr{   zwork function throw exceptionru   )r<   r-   _lockacquirer0   r   rL   rx   r}   releaserM   ry   rw   )r=   r~   r   r   lockedstartedrW   errors           r   profilez&StrobelightCLIFunctionProfiler.profile   sZ    #399AA%HH6H%%56UVV>?$d5f55--/G%%288@@B5?  '77.44<<>
12&77///E.44<<>   >N///F.44<<>	s   AD 	E6(A	E11E6)r   N)r   )r   r   r   r   r   r   boolr#   floatrk   r   listr>   rZ   rf   rm   ro   ry   r}   r   r   r   r   r   r   r   r   r   r-   r-   7   s2   
 FE
 $(/ ;,.-/6:+/ #&' ' #&	'
 ' ' '*' (+' !)c 3' d3i(' ' !'2'
RM MT M<M.!:KK 
K&D  #%b"f-#68gg#IK#	"#r   r-   profilerr   c                       st        di | dt        t        t        f   dt        t        t        t           f   f fd}|S )Nr~   r   c                      t        j                         dt        j                  dt        j                  dt
        t           f fd       }|S )Nr   r   r   c                  0     j                   g| i |S r;   )r   )r   r   r   r~   s     r   wrapper_functionz@strobelight.<locals>.strobelight_inner.<locals>.wrapper_function2  s     #8##MCDCFCCr   )	functoolswrapsr   r   r   r   r   )r~   r   r   s   ` r   strobelight_innerz&strobelight.<locals>.strobelight_inner/  sN     
	'	DBGG 	Dryy 	DXb\ 	D 
(	D  r   r   )r-   r   r   r   r   )r   r   r   s   `  r   strobelightr   )  sI     1;F; B' 	"hrl"	#  r   r;   )'r   loggingr   rR   rN   rd   collections.abcr   	threadingr   typingr   r   r   r   typing_extensionsr	   	getLoggerrL   StreamHandlerconsole_handler	Formatter	formattersetFormatter
addHandlersetLevelINFO	propagater   r   rw   r   r#   rk   r   r&   r+   r-   r   r   r   r   <module>r      sT     	 	   $  3 3 ' 
		:	;'''')GH	   Y '   / "   t_T]) 7Xc] 7c 7. . . # j jf :>56ILxB (2x|+;"<<=r   