VhddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z ddlmZddlmZmZmZmZddlmZmZmZmZmZddlZddlZddlmZddlmZdd lm Z dd l!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(dd l)m*Z*m+Z+dd l,m-Z-ddl.m/Z/erddl0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddl8mZ>ddl?m@Z@dZAdaBe-eCdZDejeCZFGddZGGddZHGd d!eIZJejd) annotationsN)IterableSequence)ThreadPoolExecutor)byrefc_size_tc_void_pCDLL)AnyCallableOptional TYPE_CHECKINGUnion)multiprocessing)get_interface_for_device) rand_strided)ir) CppCodeCache CUDACodeCache DLLWrapperget_hash PyCodeCache) get_gpu_typeis_gpu)getArtifactLogger) OrderedSet) BaseProcess)Queue) ModuleType)TritonTemplateCaller) WorkspaceArg)config)WorkspaceZeroMode) benchmarker)VCUDA_VISIBLE_DEVICESF autotuningc eZdZy)PingN__name__ __module__ __qualname__P/home/dcms/DCMS/lib/python3.12/site-packages/torch/_inductor/autotune_process.pyr*r*;r0r*c eZdZy)PongNr+r/r0r1r4r4?r2r0r4c eZdZy)!NonzeroWorkspaceNotSupportedErrorNr+r/r0r1r6r6Cr2r0r6c#pK|dytjjt}t |tjt< d|tjt=y|tjt<y#|tjt=w|tjt<wxYww)z Context manager to set the CUDA_VISIBLE_DEVICES environment variable to the specified single device. If device is None, don't manipulate the environment. N)osenvirongetr'str)devicecurrents r1set_cuda_visible_devicer>Gs ~ jjnn12G'*6{BJJ#$7 ? /0/6BJJ+ , ? /0/6BJJ+ ,sA B6B0B61B33B6ceZdZUdZdZded<dZded<dZded<dZded <e dd Z e dd Z dd Z dd Z ddZddZ d ddZddZddZdddZy) TuningProcessz Abstraction for launching a helper process to benchmark kernels. Spawns the parent process and uses multiprocessing queues to send benchmark requests and return results. N Optional[int]r<zOptional[BaseProcess]processzOptional[Queue[Any]] request_queueresponse_queuectjdtjj t  t j||y#t$rtjdYywxYw)z4 Entry point for the child process. z2Entering TuningProcess child. Visible devices = %szException in TuningProcessN) autotuning_logdebugr8r9r:r'r@workloop Exception exception)rCrDs r1 process_mainzTuningProcess.process_mainis\  @ JJNN/ 0  C  " "=. A C  $ $%A B CsAA10A1c |j}|yt|tr|jt nGt|t r |j|j ntdt|)z< Work loop for the benchmarking subprocess. NzInvalid request type ) r: isinstancer*putr4BenchmarkRequest benchmark RuntimeErrortype)rCrDobjs r1rHzTuningProcess.workloopzsr ##%C{C&""46*C!12""3==?3"%:49+#FGGr0c^|jduxr|jduxr|jduS)z? True if the sub-process has been initialized. NrBrCrDselfs r1validzTuningProcess.valids; LL $ 0""$. 0##4/ r0c.dx|_x|_|_y)z2 Reset to an uninitialized state. NrUrVs r1clearzTuningProcess.clearsCGF Ft)D,?r0c|jrytjd}|j|_|j|_|j |j|j|j f|_|jJt|j5|jjdddy#1swYyxYw)z Create child process, request/response queues, and do the warm up. Set the environment to make only the provided GPU device visible to the process. Nspawn)targetargs) rXr get_contextrrCrDProcessrKrBr>r<start)rWctxs r1 initializezTuningProcess.initializes ::< ))'2 YY[!iik{{$$""###  ||''' $T[[ 1 ! LL    ! ! !s ,CCcv|j|jJ|jj|y)z8 Push a work item to the child process. N)rcrCrN)rWrSs r1rNzTuningProcess.puts4 !!--- s#r0c|jJ|jJ |}d}|(|dk\r#|dz} |jjd} ||jj|}|S#tj$r|jj sYnwxYw|W|dk\r^#tj$r>|jj }||j|||jwxYw)a, Get a response from the child process. Raises queue.Empty on timeout or if the process dies. This method is (so far) only used by TuningProcessPool, where torch._inductor.config entries are being used to populate the timeouts: Arguments: @param result_timeout: Timeout in seconds, defaults to 120.0 or to config.max_autotune_subproc_result_timeout_seconds when called by TuningProcessPool @param graceful_timeout: Timeout in seconds to allow graceful shutdown (SIGTERM is sent after this time). Defaults to 3.0 or to config.max_autotune_subproc_graceful_timeout_seconds @param terminate_timeout: Timeout in seconds after SIGTERM, until we send SIGKILL if the process remains alive. Defaults to 1.0 or to config.max_autotune_subproc_terminate_timeout_seconds. Returns: A response from the child process (Any type) N?g?timeout)graceful_timeoutterminate_timeout) rBrDr:queueEmptyis_aliveexitcodekillrZ)rWresult_timeoutrirjremaining_timeoutresstatuss r1r:zTuningProcess.gets),||'''""... $2!'38IS8P%,%""1155c5B;--11:K1LC !;;"#||446! 7" (38IS8P;; ..>II)9*;JJL s5B+A- B+-.BB+BB+$B++AC<c|jr8|jJ|jJ|jjdyy)z8 Signal the child process to terminate. N)rXrBrCrNrVs r1 terminatezTuningProcess.terminatesH ::<<<+ ++%%1 11    " "4 ( r0cr|j+|jj|jyy)z5 Wait for the child process to exit. N)rBjoinrZrVs r1waitzTuningProcess.waits, << # LL    JJL $r0cH|j|j|jj||jjrtj d|jj |jj|jj||jjrDtjd|jj |jj|jyy)Nrgz&Sending SIGTERM to process with PID %dz&Sending SIGKILL to process with PID %d) rBrurwrmrFwarningpiderrorrorZ)rWrirjs r1rozTuningProcess.kills << # NN  LL  &6  7||$$&&&<LL$$ &&( !!*;!<<<((*"((@ ((LL%%' JJL! $r0)rC Queue[Any]rDr}returnNone)r~boolr~r)rSr r~r)g^@g@rf)r~r )g@rf)r,r-r.__doc__r<__annotations__rBrCrD staticmethodrKrHrXrZrcrNr:rurxror/r0r1r@r@\s !FM %)G ")*.M'.+/N(/C!C"C CC HH  G !2$MP1 1f)r0r@c^eZdZUdZdZded<dZded<d dZd dZd d Z dd Z dd Z y)TuningProcessPoolz Maintains a pool of TuningProcesses to benchmark kernels in parallel across devices. By default, we create one TuningProcess per device and set the sub-process environment to make only that device visible. Nz$Optional[queue.Queue[TuningProcess]] processeszOptional[ThreadPoolExecutor]executorc|jdu|jduk(sJ|jy|j}tj d|t j |_|D]R}t|}|j|jt|jj|T|jj D]$}t|jdtr$Jtt||_t s"daddl}|j%|j&yy)z, Start the child processes. Nz$Sub-process autotune device list: %s)r<)rp) max_workersTr)rrget_device_listlogrGrkrr@rcrNr*rMr:r4rlenEXIT_HANDLER_REGISTEREDatexitregisterru)rWdevicesr<prs r1rczTuningProcessPool.initialize$s $&DMMT,ABBB >> % &&( 8'B "FV,A LLN EE$&M NN  q !  "%% @Aaee4e8$? ?? @ +s7|D '&* #  OODNN + 'r0cltjsdgSt}t|}|j }t t jvrNt jt jdDcgc] }t|}}t||ksJ|Stt|Scc}w)zD Gather the list of devices to be used in the pool. N,) r#autotune_multi_devicerr device_countr'r8r9splitintrlistrange)rWgpu_typedevice_interfacecountdrs r1rz!TuningProcessPool.get_device_listIs++6M>3H= --/ 2:: -')zz2F'G'M'Mc'RS!s1vSGSw<5( ((NE%L!! Ts7B1c2|j!|jjd|_|j^|jjD]}|j |jjD]}|j d|_yy)z: Signal all child processes to terminate. N)rshutdownrrkrurx)rWrs r1ruzTuningProcessPool.terminate]s == $ MM " " $ DM >> %^^))   ^^))  !DN &r0cF|jJ|jJ|jj}|j|j |jtj tj tj|jj|S#tj$rBtjd|dtdcY|jj|SwxYw#|jj|wxYw)z Entry point for the thread-pool helper threads: Wait for an open TuningProcess, remove it from the queue, execute the benchmark in that subprocess, and return the TuningProcess to the queue. zFailed to benchmark choice 'z['. It will be ignored. Please debug the root cause in case the choice can bring perf gains.inf) bmreqrr:rNr#+max_autotune_subproc_result_timeout_seconds-max_autotune_subproc_graceful_timeout_seconds.max_autotune_subproc_terminate_timeout_secondsrkrlwarningswarnfloat)rWchoicerBs r1r]zTuningProcessPool.targetls ||'''~~)))..$$& FLL! (;;BBDDEE NN  w '{{ MM.vh7WW  <  NN  w '  NN  w 's$ Benchmark each choice in a separate process. z&Tuning process pool is not initialized)rrzipmapr])rWchoicesresultsrresults r1rPzTuningProcessPool.benchmarksp~~)S+SS)}}((("'4==+<+z*TensorMeta.from_irnodes..sAQz!Z0Afake)rlayout)fallback)r<rrrrr)rMr from_irnodesallrLayoutBuffer get_dtype get_devicerr&graphsizevars size_hintsget_sizer#unbacked_symint_fallback get_stride size_hint get_layoutrget_name)clsirnodesrrnoderr<s r1rzTensorMeta.from_irnodessU gx (>E F!1!1!!4 FF FA&AA AAM dBII &99&6D    "!!!''""-- 88.GG$$//!88077##--!((88.  !GsFct|j|j|j|j|j S)N)r<r extra_size)rrrr<rrrVs r1 to_tensorzTensorMeta.to_tensors2 JJ LL;;**{{   r0)rz/Union[LayoutOrBuffer, Sequence[LayoutOrBuffer]]r~#Union[TensorMeta, list[TensorMeta]])r~ torch.Tensor)r,r-r.rr classmethodrrr/r0r1rrsQ   (( ++ KD-! E! ,! ! F r0rcxeZdZdZ d dZ d dZd dZdd d dZdd d dZy) rOa1 Only handle triton template benchmark for now. The extern kernel benchmark can be done inside the same process since they usually don't cause crash. Important: Instances of this class and subclasses have to be serializable across process boundaries. Do not put CUDA Tensors in here! c||_t|tr|g}||_ttt fr)t dkDrtfdDsJd|_||_ y)Nr!c3dK|]'}dD] }td|t||k(")yw))r<rrrrrN)getattr)rrattroutput_tensor_metas r1rz,BenchmarkRequest.__init__..sG Q.q148GAt.s!PA!++-!Prrz0Skipping op due to nonzero workspace requirementrz6InChildProcess %s: load %f, create tensor %f, bench %f)rF isEnabledForloggingDEBUGtimerrrrrrr6inforrrGr;r) rWrrrGstart_tscreate_tensor_elapser load_elapseout bench_elapses r1rPzBenchmarkRequest.benchmark sQ ++GMM: yy{H  }%* **!!P9O9O!PPM 33==?M #'99;#9 yy{H !!!=N NB ))+0Kyy{HdmmB>> > 99;1L  HD $    +1    R S<  s0E)E=<E=) rr;rrrrr Iterable[Any]r~rrrrrr~zCallable[[], None]rrrrzOptional[torch.Tensor]r~r) r,r-r.rrrrrrPr/r0r1rOrOs%%?%@ % " %  %6"*";G" "  15 "%". "  "15)$).)  )r0rOc2eZdZdZdddZdd ddZy) TestBenchmarkRequestz Supports unit testing. Defined in this file so that the TuningProcess sub-process knows how to unpickle these objects. Nc||_yr)value)rWrs r1rzTestBenchmarkRequest.__init__?s  r0rcH|j td|jS)Nz Failed to run)rrIrs r1rPzTestBenchmarkRequest.benchmarkBs# :: O, ,zzr0r)rzOptional[float]r~rr)r,r-r.rrrPr/r0r1rr9s0 UY*;Q r0rc$eZdZdd ddZy)GPUDeviceBenchmarkMixinNrctdg||D}t|dks Jd|td|Dd}t|}t|dk(rtt |}n|j }|j |5tj|}|jddd|S#1swYSxYw)Nc3K|]i}t|tjrMt|jj r.|jj |jj kywr)rMtorchTensorrr<rRindexrtensors r1rz3GPUDeviceBenchmarkMixin.do_bench..QsR$ &%,,/v}}))* ##/ MM  $ sA/A1r!zCan not mix devices c3K|]9}t|jjr|jj;ywr)rr<rRrs r1rz3GPUDeviceBenchmarkMixin.do_bench..Zs4 &--,,- "" s?Acuda) rrnextritercurrent_devicer<r% benchmark_gpu synchronize) rWrrrdevice_idx_set device_typer device_idxrs r1rz GPUDeviceBenchmarkMixin.do_benchKs $$ 9M9=9$  >"a'P+??O)PP' +    4K@ ~ ! #d>23J)88:J  $ $Z 0 +++B/C  ( ( * +  + s &CC rr,r-r.rr/r0r1r r Js* 15 %.   r0r c$eZdZdd ddZy)CPUDeviceBenchmarkMixinNrc,tj|Sr)r% benchmark_cpurs r1rz CPUDeviceBenchmarkMixin.do_benchns ((,,r0rrr/r0r1rrms* 15 -%-. -  -r0rc~eZdZ d dfd Z ddZdZddZxZS) TritonBenchmarkRequestc t |||||||_||_||_||_| |_| |_| |_| |_ yr) superr module_pathmodule_cache_key num_stages num_warpsmatrix_instr_nonkdim waves_per_eukpack workspace_arg)rWrrrrr%r&r'r(r)r*r+r, __class__s r1rzTritonBenchmarkRequest.__init__zsW &79KZX& 0$"$8!( *r0c tj|j|j}tj d|j|jt ||jj t|jd j_ i ddl }d|j jvrd d<j j"dk(rd nPj j"}t%|}|j'|j(j j* |j,|j,  fd}|St/t ||jt0j2j4j6j8r!t;j< gi d iSt;j< gi  dd S) Nz"benchmark module key: %s, path: %sFrwarmupcpucj}tj|fdtjj}j t jk7r|jg|iddy)N)r!rr<Tstream benchmark_run) rr  empty_strideduint8r< zero_moder$ UNINITIALIZEDzero_) workspace_sizeworkspace_tensorrrr run_methodr4 warmup_argr,s r1run_with_workspacez>TritonBenchmarkRequest.make_run_fn..run_with_workspaces!.!4!4#(#6#6#%++(// $ !**.?.M.MM$**,"!%  !  ""&r0r4Tr3)rload_by_key_pathr&r%rFrGrrrunrr__self__with_bandwidth_infoinspect signature parametersr<rRrget_raw_streamrrr,rMr  _inductorruntimetriton_heuristicsDebugAutotuner functoolspartial) rWrrmodrDrrr?rr=r4r>r,s `` @@@@@r1rz"TritonBenchmarkRequest.make_run_fns**4+@+@$BRBRS 0  ! !    S$"2"2377 $//* 27 /  w((4?? ?#(Jx    $ $ -F'..33K7 D %44''..44F    ) ..M  2& %  C)) * OO # # 5 5 D D $$      $$    " r0ctj|j|j}t ||j j yr)rr@r&r%rr precompile)rWrNs r1rPz!TritonBenchmarkRequest.precompiles9**4+@+@$BRBRST%%&113r0cTd|jd|jd|jS)Nself.kernel_name=z, self.module_path=z, self.module_cache_key=)rr%r&rVs r1__str__zTritonBenchmarkRequest.__str__s2#$""$$8t'7'7&99RDN>NPT*U' t'r0ctjd|tj|jdtjd|y)NPrecompiling %sr_Done precompiling %s)rFrGrcompiler`rVs r1rPzCUDABenchmarkRequest.precompile s< .5d..53T:r0c B|j|jt||gzDcgc]}t|j }}t j d|j|j|j|j||jttjjj}t!|j|j}td}|j"dkDrhtj$|j"dzdztj&|j(|_t|j*j }t-j.|g||jd||Scc}w)Nzqmake_run_fn: self.kernel_name=%s, self.source_file=%s, self.hash_key=%s, self.DLL=%s, args=%s, self.extra_args=%srr2)ensure_dll_loadedupdate_workspace_sizerr data_ptrrFrGrrerdrbrr rcurrent_stream cuda_streamrr;zerosfloat64r<rarLrM)rWrrrr^ stream_ptrr= workspace_ptrs r1rz CUDABenchmarkRequest.make_run_fnsy   ""$}-?  V__& '           MM HH  OO ejj779EEF TXXt'7'78      ""[[$$q(Q.mm$++DN %T^^%<%<%>?M    __         3 s Fc  |jry|jt|jDchc]}|jc}}t |dzDcgc] }t d}}t tjjj}t|j|j}t}|g||jt!|d|tjj#|j$|_t(j+d|j&|j|j,|j.|j||jd|_ycc}wcc}w)Nr!zupdate_workspace_size called: new workspace size=%d, self.kernel_name=%s, self.source_file=%s, self.hash_key=%s, self.DLL=%s, args=%s, self.extra_args=%sT)rcrorrrrr r rrrrsrrbrrrrrrr;rFrGrerd)rWmetaunique_input_count_r^rvr=c_workspace_sizes r1rpz*CUDABenchmarkRequest.update_workspace_size;sW  ' '    #'#9#9 :4TYY : )..@1.D(EF1FFejj779EEF TXXt'7'78 #:  __            .44 h          MM HH  OO (,$; ;Fs E<Fc|j4tj|jd\|_|_|_yy)Nr_)rbrloadr`rdrerVs r1roz&CUDABenchmarkRequest.ensure_dll_loaded_s: 88 8E8J8J  $9 5DHdmT%5 r0c^|j|jjd|_yr)rbcloserarVs r1rz#CUDABenchmarkRequest.cleanup_run_fnes! 88  HHNN r0cTd|jd|jd|jS)NrRz, self.source_file=z, self.hash_key=)rrerdrVs r1rSzCUDABenchmarkRequest.__str__js0#$""$$8t'7'7&99JDMM;KLLr0 rr;rrrrrrr`r;r~rrrrT) r,r-r.rrPrrprorrSrUrVs@r1r\r\sVV?V@ V " V  V V$;% *% ;G% % N",H  Mr0r\cbeZdZ dfd ZdZ ddZddZd dZxZS) CppBenchmarkRequestcft|||||||_t||_d|_yr)r$rr`rrdrbrgs r1rzCppBenchmarkRequest.__init__rs5 &79KZX& - 6:r0ctjd|tj|jdtjd|y)Nrir0rrj)rFrGrr~r`rVs r1rPzCppBenchmarkRequest.precompiles< .5$**>3T:r0c\tj|jd|_t ||gzDcgc]}|j }}t jd|j|j||jt|j|j}td|jDsJtjgt|tt |jzz|_t!j"|g||jScc}w)Nr0rzJmake_run_fn: self.kernel_name=%s, self.DLL=%s, args=%s, self.extra_args=%sc3PK|]}t|tj ywr)rMctypes c_ulonglong)rargs r1rz2CppBenchmarkRequest.make_run_fn..sR3:c6#5#56Rs$&)rr~r`rbrrqrFrGrrrrrrrargtypesrLrM)rWrrrr^r=s r1rzCppBenchmarkRequest.make_run_fns $$T%5%55I04]0C}o0UVf!VV X    HH  OO  TXXt'7'78 R$//RRRR%112 ID12 2      __  !WsD)c|j3 t|jdr|jjyyy)Nr)rbhasattrrrVs r1rz"CppBenchmarkRequest.cleanup_run_fns9 88  txx) * r0c d|jS)NrR)rrVs r1rSzCppBenchmarkRequest.__str__s#$""$%%r0rrrrT) r,r-r.rrPrrrSrUrVs@r1rrnso ; ;? ;@ ; " ;  ;  ;; * ;G  6!&r0rc,tj|S)zO Do benchmarking in a subprocess and return the perf number (latency). ) tuning_poolrP)rs r1benchmark_in_sub_processrs   ))r0)r<rAr)_ __future__r contextlibr dataclassesrLrr8rkrrcollections.abcrrconcurrent.futuresrrrr r typingr r r rrr torch._inductor.async_compilertorch._dynamo.device_interfacertorch._dynamo.testingrtorch._inductorrtorch._inductor.codecacherrrrrtorch._inductor.utilsrrtorch._loggingrtorch.utils._ordered_setrmultiprocessing.processrmultiprocessing.queuesrtypesr torch._inductor.select_algorithmr codegen.commonr"r^r#r$runtime.benchmarkingr% virtualizedr&r'rr,rF getLoggerrr*r4rIr6contextmanagerr> dataclassr@rrrrLayoutOrBufferrrOrr rr"rXrZr\rrr/r0r1rs"  .122@@ $!C.7,/3, E,--."8\:g!       77( yyyx ~~~B ! ryy"))+, 3 3 3 l ]]]@+"  F--uU-uUp 79O  79O tM24DtMn<&13C<&~* '*&*r0