
    Vh-                     B   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZ d dlmZ  ej(                  d      Z ej,                         Z ej0                  d      Zej5                  e       ej7                  e       ej9                  ej:                         d	e_         ed
      Z ed      Z  G d de!      Z"ddee#   de$fdZ%ddee#   de#fdZ&dee$   de$fdZ' G d d      Z(	 ddee(   dedeeee f   geeee    f   f   fdZ)y)    N)Sequence)Lock)default_timer)AnyCallableOptionalTypeVar)	ParamSpecstrobelight_function_profilerzB%(name)s, line %(lineno)d, %(asctime)s, %(levelname)s: %(message)sF_P_Rc                       e Zd ZdZy)StrobelightCLIProfilerErrorzC
    Raised when an error happens during strobelight profiling
    N)__name__
__module____qualname____doc__     X/home/dcms/DCMS/lib/python3.12/site-packages/torch/_strobelight/cli_function_profiler.pyr   r       s    r   r   pidreturnc                 ~    d}| xs t        j                         } t        j                  |j                  |             S )zFReturns the link to the process's namespace, example: pid:[4026531836]z/proc/{}/ns/pid)osgetpidreadlinkformat)r   PID_NAMESPACE_PATHs     r   _pid_namespace_linkr   &   s3    *

C;;)00566r   c                     | xs t        j                         } t        |       }t        ||j	                  d      dz   d       S )z"Returns the process's namespace id[   )r   r   r   intfind)r   links     r   _pid_namespacer'   -   s;    

Cs#DtDIIcNQ&,--r   commandc                 $    dj                  |       S )N )join)r(   s    r   _command_to_stringr,   4   s    88Gr   c                      e Zd ZdZ e       Zddddddddddd	
d
ededede	dedede
ee	      de
ee	      dedefdZd"dZd#deddfdZd"dZd"dZdeddfdZdefdZdeeef   dej,                  d ej.                  de
e   fd!Zy)$StrobelightCLIFunctionProfilera  
    Note: this is a Meta only tool.

    StrobelightCLIFunctionProfiler can be used to profile a python function and
    generate a strobelight link with the results. It works on meta servers but
    does not requries an fbcode target.
    When stop_at_error is false(default), error during profiling does not prevent
    the work function from running.

    Check function_profiler_example.py for an example.
    FiX  g    cAzpytorch-strobelight-ondemand<   N   )
stop_at_errormax_profile_duration_secsample_eachrun_user_nametimeout_wait_for_running_sectimeout_wait_for_finished_secrecorded_env_variablessample_tagsstack_max_lenasync_stack_max_lenr1   r2   r3   r4   r5   r6   r7   r8   r9   r:   c       
             || _         || _        || _        || _        || _        || _        d | _        d | _        || _        y N)	r1   r2   r3   r4   r5   r6   current_run_idprofile_resultr8   )selfr1   r2   r3   r4   r5   r6   r7   r8   r9   r:   s              r   __init__z'StrobelightCLIFunctionProfiler.__init__H   sN     +(@%&*,H)-J* .237&r   r   c                    t        j                         }t        |      }ddddddddt        | j                         d	t        | j
                  d
z         d| d| g}| j                  r;|j                  d       |j                  dj                  | j                               t        j                  dt        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                   dk7  rt#        d|       t%        j&                  d|      x}r t        |j)                  d            | _        y t#        d|       )Nstrobeclientrunz
--profilerpyperfz--eventcyclesz--asyncz--sample-intervalz--duration-msi  z--pid:z--sample-tags,running command: %sTcapture_outpututf-8output:
{%s}r   z:failed to start strobelight profiling, error in run_async:zINFO Run Id: (-?\d+)r"   z9failed to start strobelight profiling, unexpected result )r   r   r'   r$   r3   r2   r8   appendr+   loggerdebugr,   
subprocessrC   stderrdecode
returncoder   researchgroupr=   )r?   	processId	namespacer(   resultoutputmatchs          r   
_run_asyncz)StrobelightCLIFunctionProfiler._run_asyncb   sW   IIK	"9-	4##$%4004789k9+&
  NN?+NN388D$4$456*,>w,GH=%%g._f-!-LVHU  II5v>>5>"%ekk!n"5D)GxP
 	
r   counterc                 D   |dkD  rt        d      ddd| j                   g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d	      }t        j                  d
|       |j                  dk7  rt        d|       t        j                  d|      x}rU|j                  d      }|dk(  ry |dk(  r*t        j                  d       | j                  |dz          y t        d| d      t        d| d      )N   z*wait_for_running called more than 20 timesrB   getRunStatus--run-idrH   TrI   rK   rL   r   zAfailed to start strobelight profiling, error in wait_for_running:zProfile run status: (.*)r"   RUNNING	PREPARING
   zunexpected z phaseunexpected output
: r*   )r   r=   rN   rO   r,   rP   rC   rQ   rR   rS   rT   rU   rV   timesleep_wait_for_running)r?   r]   r(   rY   rZ   r[   current_statuss          r   rh   z0StrobelightCLIFunctionProfiler._wait_for_running   s#   R<-<  ">:$BUBUAVX*,>w,GH=%%g._f-!-STZS[\  II8&AA5A"[[^N*;.

2&&w{31K?Ov2VWW),A&*KLLr   c                    dddt        | j                        g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                  d	k7  rt        d
|       t        j                  d|      x}r2|j                  d      }|j                  d      ry t        d| d      t        d| d      )NrB   stopRunra   rH   TrI   rK   rL   r   z<failed to stop strobelight profiling, return code is not 0 :INFO ::1:(.*)r"   zSuccess!z*failed to stop strobelight profiling, got z resultre   r*   )strr=   rN   rO   r,   rP   rC   rQ   rR   rS   r   rT   rU   rV   __contains__)r?   r(   rY   rZ   r[   ri   s         r   	_stop_runz(StrobelightCLIFunctionProfiler._stop_run   s    !9j#d>Q>Q:RS*,>w,GH=%%g._f-!-NvhW  IIov6656"[[^N**:61@@PPWX  *,A&*KLLr   c                    dddt        | j                        g}t        j                  dt	        |             t        j                  |d      }|j                  j                  d      }t        j                  d|       |j                  d	k7  rt        d
|       t        j                  d|      x}rg|j                  d      }|j                  d      r&t        j                   d       | j#                          y |j                  d      st        d|       g | _        t        j&                  d|      D ]2  }| xj$                  |d	   z  c_        t        j)                  |d	          4 y )NrB   r`   ra   rH   TrI   rK   rL   r   z<failed to extract profiling results, return code is not 0 : rl   r"   zProfile run status: PROCESSINGrd   z!Profile run finished with SUCCESSz9failed to extract profiling results, unexpected response zF(Total samples(.*)|GraphProfiler(.*)|Icicle view \(python stack\)(.*)))rm   r=   rN   rO   r,   rP   rC   rQ   rR   rS   r   rT   rU   rV   rn   rf   rg   _get_resultsr>   findallinfo)r?   r(   rY   rZ   r[   ri   items          r   rq   z+StrobelightCLIFunctionProfiler._get_results   sK   !>:s4CVCV?WX*,>w,GH=%%g._f-!-NvhW  IIov6656"[[^N**+KL

2!!##001TU1OPVxX  !JJU
 	!D 47*KKQ 	!r   collect_resultsc                     	 | j                          t        j                  d       t        j                  d       |sy | j	                          y # t
        $ r t        j                  dd       Y y w xY w)Nzstrobelight profiling stoppedzcollection stoppedzerror during stop_strobelightTexc_info)ro   rN   rs   rO   rq   	Exceptionwarning)r?   ru   s     r   _stop_strobelight_no_throwz9StrobelightCLIFunctionProfiler._stop_strobelight_no_throw   s`    	KNNKK78LL-." 	KNN:TNJ	Ks   <A A  A32A3c                 ,   d}	 | j                          d}t        j                  d| j                         | j	                          t        j                  d       y# t
        $ r. t        j                  dd       |r| j                  d       Y yw xY w)NFTzstrobelight run id is: %szstrobelight profiling runningzerror during start_strobelight:rw   ru   )r\   rN   rs   r=   rh   ry   rz   r{   )r?   strobelight_starteds     r   _start_strobelightz1StrobelightCLIFunctionProfiler._start_strobelight   s    #	OO"&KK3T5H5HI""$KK78 	NN<tNL"///F		s   AA 4BBwork_functionargskwargsc                 `   d | _         d | _        t        j                  j	                  d      x}r$|s4| j
                  rt        d      t        j                  d        ||i |S | j                         }|s]| j
                  r)t        j                  j                          t        d       ||i |}t        j                  j                          |S 	 t        j                  d       t               } ||i |}t               }||z
  }	t        j                  d|	       | j                  d       t        j                  j                          |S y # t        $ rN}
t        j                  dd	       | j                  d       t        j                  j                          |
d }
~
ww xY w)
NFzconcurrent runs not supportedz%failed to start strobelight profilingzcollection startedzwork function took %s secondsTr}   zwork function throw exceptionrw   )r=   r>   r.   _lockacquirer1   r   rN   rz   r   releaserO   timerrs   r{   ry   )r?   r   r   r   lockedstartedrY   startend
total_timeerrors              r   profilez&StrobelightCLIFunctionProfiler.profile  s    #"399AA%HH6H%%56UVV>?$d5f55--/G%%288@@B5?  '77.44<<>12&77g 5[
;ZH///E.44<<>   >N///F.44<<>	s   A=E 	F-A	F((F-)r   N)r   )r   r   r   r   r   r   boolr$   floatrm   r   listr@   r\   rh   ro   rq   r{   r   r   r   r   r   r   r   r   r   r   r.   r.   8   s2   
 FE
 $(/ ;,.-/6:+/ #&' ' #&	'
 ' ' '*' (+' !)c 3' d3i(' ' !'4'
RM MT M<M.!>KK 
K&D  (%b"f-(68gg(IK(	"(r   r.   profilerr   c                       st        di | dt        t        t        f   dt        t        t        t           f   f fd}|S )Nr   r   c                      t        j                         dt        j                  dt        j                  dt
        t           f fd       }|S )Nr   r   r   c                  0     j                   g| i |S r<   )r   )r   r   r   r   s     r   wrapper_functionz@strobelight.<locals>.strobelight_inner.<locals>.wrapper_function;  s     #8##MCDCFCCr   )	functoolswrapsr   r   r   r   r   )r   r   r   s   ` r   strobelight_innerz&strobelight.<locals>.strobelight_inner8  sN     
	'	DBGG 	Dryy 	DXb\ 	D 
(	D  r   r   )r.   r   r   r   r   )r   r   r   s   `  r   strobelightr   2  sI     1;F; B' 	"hrl"	#  r   r<   )*r   loggingr   rT   rP   rf   collections.abcr   	threadingr   timeitr   r   typingr   r   r   r	   typing_extensionsr
   	getLoggerrN   StreamHandlerconsole_handler	Formatter	formattersetFormatter
addHandlersetLevelINFO	propagater   r   ry   r   r$   rm   r   r'   r,   r.   r   r   r   r   <module>r      sW     	 	   $  ) 3 3 ' 
		:	;'''')GH	   Y '   / "   t_T]) 7Xc] 7c 7. . . # r rv :>56ILxB (2x|+;"<<=r   