Vhr ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z ddl mZddlZddlZddlmZddlmZddlmZdd lmZmZdd lmZd d lmZd d lm Z d dl!m"Z"d dl#m$Z$m%Z%d dl&m'Z'm(Z(m)Z)d dl*m+Z+m,Z,m-Z-ddl.m/Z/m0Z0m1Z1m2Z2m3Z3iejhdejjdejldejndejpdejrdejtdejvdejxdejzdej|dej~dejd ejd!ejd"ejd#ejd$ejd%ejd&ejd'iZHiejhd(ejjd)ejld*ejnd+ejpd,ejrd-ejtd.ejvd/ejxd0ejzd1ej|d2ejxd0ejvd/ej~d3ejd4ejd5ejd6ejd7ejd8ejd9ejd:ejd;iZIdd?d@ZJejdAejdBiZMejdCk(ZOdZPedDgdEZQdFZRdGZSdHejjdIe3fdJZVdIe3fdKZWdLZXGdMdNe/ZYGdOdPeZejZ[dQZ\dRZ]dSdTdUejdVe_fdWZ`dSdTdUejdVe_fdXZaGdYdTe-jZcGdZd[Zdejfd\e'd]efe/d^ffd_Zgd`ZhejhfdaZidbZjdcZkddZldeZmdfe$dgene$dIefe?e?ffdhZoy)iN) namedtuple)Sequence)AnyCallableOptional)patch)is_integer_dtype) OrderedSet) CppPrinter)symbol_is_typeSymT) ValueRanges)ir)Dep)LoopBody)BaseSchedulerNodeSchedulerBuffer)IndentedBuffersympy_index_symbol_with_prefix sympy_subs)opsOpsValueV) CSEVariablededuce_output_dtype_by_nameKernel KernelArgsOptimizationContextfloatdoublehalfint64_tint32_tint16_tint8_tuint64_tuint32_tuint16_tuint8_tboolbfloat16zc10::complexzc10::complexzc10::complex float8_e4m3fn float8_e5m2float8_e4m3fnuzfloat8_e5m2fnuzz at::kFloatz at::kDoublez at::kHalfz at::kLongzat::kIntz at::kShortz at::kCharz at::kUInt64z at::kUInt32z at::kUInt16z at::kBytez at::kBoolz at::kBFloat16zat::kComplexHalfzat::kComplexFloatzat::kComplexDoublezat::kFloat8_e4m3fnzat::kFloat8_e5m2zat::kFloat8_e4m3fnuzzat::kFloat8_e5m2fnuzz at::kMetazat::kCPUz at::kCUDAzat::kXPU)metacpucudaxpuz at::kStridedz at::kMkldnnwin32 GemmBlocking)block_mblock_nblock_kc td|DrMtjtj|Dcgc]}t |t s|j!c}SdScc}w)Nc3ZK|]#}t|ts|jdu%ywN isinstanceCppCSEVariabledtype).0ns Q/home/dcms/DCMS/lib/python3.12/site-packages/torch/_inductor/codegen/cpp_utils.py z$get_promote_dtype..ms"RqJq.4Qqwwd"R++)all functoolsreducetorch promote_typesr?r@rA)argsrCs rDget_promote_dtyperMgs[ RDR R    " DjN&CQWW D    Es A" A" cd}t|}tj||}td|Dr|rt t ||}|S)Nct|trX|jrL|rJ|j|k7r;tj||}t|t r |j n|}||_|Sr=)r?r@rArto_dtypervalue)arg promote_types rD promote_argz!promote_args..promote_argssR sN +  \),,sL1C)#x8#))cC$CI )rSc3XK|]"}t|tr|jdu$ywr=r>)rBnew_args rDrEzpromote_args..s+ '>2 MM % s(*)rMrHpartialrGlistmap)new_argsrTrS promote_fns rD promote_argsr]rs[ %X.L""!J  #  J12 OrUnodereturncV|jjtjdSr=)r2getr key)r^s rD get_opt_ctxrcs 99==,00$ 77rUctjjsJttjjSr=)r interpreter current_nodercrUrDget_current_node_opt_ctxrhs* == % %% % q}}11 22rUc t|g|i|x} |S|dk(rmttjdr=tjjj j dr tJtjStd|DsJtjtj|Dcgc]}t|ts|j!c}Scc}w)Nmaskedrfmasked_subblockc3ZK|]#}t|ts|jdu%ywr=r>rBrRs rDrEz4deduce_dtype_for_cpp_cse_variable..s' &)Z^5TCIIT ! rF)rhasattrrrerftarget startswithrhrArGrHrIrJrKr?r@)namerLkwargs output_dtyperRs rD!deduce_dtype_for_cpp_cse_variablerts3         AMM> 2 **11<<=NO(*6  7()/// -1       "& J3*S.*ISYY J  Js =C( C( ceZdZ d deedeejddffd Zde fdZ dZ de jfd Zd e jfd ZxZS) r@NboundsrAr_cvt||||d|_ttj |_y)NF)super__init__is_vecr sympySymboldependent_itervars)selfrqrvrA __class__s rDryzCppCSEVariable.__init__s1 vu- ",U\\":"<rUc d|jd|jd|jd|jd|jd S)NzCppCSEVariable(name: z , bounds: z , is_vec: z , dtype: z, dependent_itervars: ))rqrvrzrAr})r~s rD__repr__zCppCSEVariable.__repr__sQ#DII;j ZPTP[P[}\efjfpfpeqr##'#:#:";1 > rUc |dk(r|j|dns|jj|Dcgc]}t|tr |j c}|dk(r|j|dt d|Drd|_|jt|g|i||_|jJycc}w)Nloadr index_exprrc3VK|]!}t|ts|j#ywr=)r?r@rzrms rDrEz0CppCSEVariable.update_on_args..sQ#C1P3::Qs))T) _set_dependent_itervarsr}updater?r@anyrzrArt)r~rqrLrrrRs rDupdate_on_argszCppCSEVariable.update_on_argss 6>  ( (a 1 +D # # * * $!#~6** |#,,T!W5QQQ" :: ;4Q$Q&QDJzz%%%!s#B=indexc|jD]}|tjjvr|jj |;|j tjjjvsl|jjtjjj|j jy)z Set the relevant itervars for this variable based on the `index` expression. This includes the itervars directly used in the `index` as well as relevant itervars of other cse variables used in the `index`. N) free_symbolsrkernelitervarsr}addrqcse varname_mapr)r~rss rDrz&CppCSEVariable._set_dependent_itervarss ## AAHH%%%''++A.188<<333''..HHLL,,QVV4GG  rUitervarc||jvSr=)r})r~rs rD depends_onzCppCSEVariable.depends_ons$1111rUr=)__name__ __module__ __qualname__rrrrJrArystrrrr{Exprrr|r __classcell__rs@rDr@r@sj (, =C = $ =  = # &2 UZZ 2%,,2rUr@c,eZdZddddeffdZxZS)r T)simplifyprc|r]t|tjrCttj dr)tj j j|}t|%|S)Nsizevars) r?r{rrnrgraphrrrxdoprint)r~exprrrrs rDrzCppPrinter.doprintsI  44*9U77##,,T2Dwt$$rU)rrrr,rrrs@rDr r s04%%%rUr c.dtdt|dS)N static_cast<>(r) INDEX_TYPEcexprrs rD cexpr_indexrs *Re ~Q 77rUc |tdk(rd|dS|tdk(rd|dSt|tr d|dt|j dSt j |rd|d Sd|dt|dS) Nz-infz-std::numeric_limits::infinity()infzstd::numeric_limits::quiet_NaN())r!r?r,rlowermathisnanrepr)rQcpp_types rD value_to_cpprs f &xj >> %, %hZ}== E4 hZr#e**:*:*<)=Q?? E %hZ~>>hZr$u+a88rUlocalize_buffer_handlerLocalizeBufferHandlerrglobal_buf_namectjjj|j}|J|j |}|j }t|dj\}\}}t|t|z} tt|jjD cgc]} dt| | dzz } } t|jd} i} | D]X}|j j#ds|j | vs.t$j&j(j+| |<Zt-|| }|Scc} w)Nc4t|jSr=)int is_reduction)xs rDz,rewrite_index_for_function..ss1>>+;'<rU)rbrrc|jSr=)rq)rs rDrz,rewrite_index_for_function.."s affrU)rr scheduler name_to_buf defining_opglobal_to_local get_nodesmaxgrouptuplerangelen get_layoutsizesortedrrqrpr{corenumbersZeror)rrrsnode local_bufscheduler_nodes_rreduction_group call_rangesidxindices_to_keepsorted_symbols replacementsrs rDrewrite_index_for_functionrsJ GG   ) )/ : F FE   '77HIoo'O"%<# e A,!77KY113889:  C a ( )*OE..4DENL 8 66  S !affO&C#jj00557LO8 ul +E Ls8EcXtd|jD}g}|j|}tt |j D]3}t tj|}|j||vr|nd5|jj|}|S)Nc3VK|]!}t|tjs|#ywr=)r r INDEX)rBrs rDrEz*rewrite_index_for_nodes..1s"4::)Fs))r) r rrrrget_sizerr rappendr make_indexer)rrr used_vars index_varsrivars rDrewrite_index_for_nodesr,s %%IJ'77HI 3y))+, -:,TZZ; !1#q9: 2I " / / 1* =E LrUceZdZdeeej fdedejegejfddffd Z dedejfdZ dedejfd Z d d Z d ZxZS) rr rewrite_indexr_Nc@t||||_||_yr=)rxryrr)r~innerrrrs rDryzLocalizeBufferHandler.__init__>s" .*rUrqrc|jrL||jvr>|jJ|j|||}|j|j}||fSr=)rrget_namer~rqrs rDlocalizezLocalizeBufferHandler.localizeHsa   DD,@,@$@%%1 11&&tUD9E''-668DU{rUcT|jj|j||Sr=)_innerrrrs rDrzLocalizeBufferHandler.loadOs$t{{tU!;<.get_dtype}s6t)))))$/99;;%d+ +rUrc4|jvr|S|Sr=r)rqoriginal_inputr~s rDinputz+LocalBufferContext.__enter__..inputs!t))) !$' 'rUr c4|jvr|S|Sr=r )rqoriginal_outputr~s rDoutputz,LocalBufferContext.__enter__..outputs!t))) "4( (rUr) r __enter__rrr enter_contextrobjectrr rset_local_buffer_context)r~rr rrr rs` @@@rDrzLocalBufferContext.__enter__ys !!#WW.. , %%ell177K&ST))// ( %%ell43C3CWe&TU**11 ) %%ell43C3CXv&VW %%a&@&@&FG rUcr|jj|jj|||yr=)rclearr__exit__)r~exc_typeexc_valexc_tbs rDrzLocalBufferContext.__exit__s*   "   7F;rU local_bufferrc|j|jvsJ||j|j<|r|D]}|j}||jvr||jvsJ||j|<||j|<|tj j vsl|j j|tj j j|yyr=)rrrrrrrr)r~rr global_bufferglobal_buffer_names rDadd_local_bufferz#LocalBufferContext.add_local_buffers$$&d.@.@@@@6B<0023 !/ D %2%;%;%="&d.A.AA*$2F2FFG;H##$67;G$$%78%QWW-D-DD((,,-?@GG++//0BC D rUfn.rrcfd}|S)Nctjttjj5|i|cdddS#1swYyxYw)N)rr)rset_ops_handlerrget_ops_handlerr)rLrrrrr~s rDrz3LocalBufferContext.localize_function..innersU""%%%'$($8$8"/ +4*6* + + +s AArg)r~rrrs``` rDlocalize_functionz$LocalBufferContext.localize_functions + rUnodesct|dkDsJdtjffd }|Dcgc] }|| c}Scc}w)aX Given `local_buf` and `global_buf` registered in current `LocalBufferContext` though the method of `add_local_buffer`, localizes the `global_buf` to `local_buf` for the given `nodes` and returns a new list of IR nodes that work on `local_buf` instead of `global_buf`, i.e., all the loads and stores are redirected to `local_buf`. This helps the fused loops to work on smaller-sized local buffers for better data locality. The the data access of `local_buf` is assumed to be contiguous with the same order as the `global_buf`. rr^ct|tjr |jn|}t|tjsJj |j }tj||}t|tjr6tj|j|j|}|S|}|S)N)inner_fn)rqlayoutdata) r?rComputedBufferr*Loopsr$r( dataclassesreplacerr)r^loops new_inner_fn new_loopsnew_noderr~s rDwrap_inner_fn_for_nodezALocalBufferContext.localize_nodes..wrap_inner_fn_for_nodes!+D"2C2C!DDII$EeRXX. ..11L $++ELII$ 1 12,,1B O%OrU)rrIRNode)r~r%rr3r^s` ` rDlocalize_nodesz!LocalBufferContext.localize_nodessA$5zA~~  $:??&t,???sAr=)rrr__doc__rryrrrrrrYrrrrr{rrr$rr4r5rgrUrDrrcs =J =4 = D< TXDIID7?RYY7PD4 '  S#X  $ejj# 6 B 0 $ &@BII&@ $ejj# 6 B &@ bii &@rUrbuffervars.c"fd|D}|S)zk Given list of cse variables, Cast each to new mask base dtype and return casted cse variable. c3K|]M}tjjjtjj |Oywr=)rrrgenerate_get_mask_cast)rBrr7rAs rDrEz'unify_mask_base_type..sH    xx&&sE23 sAArg)r7r8rAnew_varss` ` rDunify_mask_base_typer>s  H OrUc|jtjk(rV|jtjk(sJtj}t t j j||f|S||fS)zx Given two cse variables, when dtype is bool, unify them to the same mask dtype and return casted cse variable. )rArJr,int32r>rrcompute)ab mask_dtypes rDmay_unify_binary_op_mask_typerEsY ww%**ww%**$$$[[ #AHH$4$4q!fjII a4KrUct|jsJ|jd|j5|jt|jdt j jd|jt|dt j jd|j|d|jdt|jdt j jd|j5|j|dddt j j| }|d k(r|jd t|d n|jd t|d|d ddd|jd|S#1swYxYw#1swY(xYw)Nz[&]()z offset[z];z result[z.store(offset);zfor( z offset_idx = 0; offset_idx < z; offset_idx++ ))rArzreturn at::vec::Vectorized::loadu(result);zreturn at::vec::VectorizedN.inner_fn5s. 'E<<5)D;;ud+ +rUgelu algorithmnonec|}tjk7r$tj|tj}tjdtj}tjdtj}tjdtj}||ztj ||z|zz}tjk7rtj|}|S)N??g;f?)rJr!rrPr\erf)rr r#oneconstresultrAr_s rDr(z+create_epilogue_with_attr..inner_fn>s$U+EKK'LL .inner_fnMs$U+EKK'LL .inner_fncs% 'ES[[//FMrUrqc:tj|Sr=rprr_s rDr(z+create_epilogue_with_attr..inner_fnjs;;|E23 3rUc:tj|Sr=)rrjrss rDr(z+create_epilogue_with_attr..inner_fnos88L/0 0rU hardswish hardsigmoidctjdtj}tjdtj}tjdtj}tjdtj}tj||z|}tj ||}||zS)NrgUUUUUU?)rr\rJr!r]minimum)r r^sixthree one_over_sixrmins rDhardsigmoid_floatz4create_epilogue_with_attr..hardsigmoid_floatts<<5;;/D,,q%++.CLLEKK0E<<(;U[[IL++eemT2C++c3'C% %rUc|}tjk7r$tj|tj}|}dk(r||z}tjk7rtj|}|S)Nru)rJr!rrP)rr riattrrArr_s rDr(z+create_epilogue_with_attr..inner_fn}si 'E # UEKK8&u-F{" #fe4MrU leaky_reluscalarsrrc |}tjk7r$tj|tj}tjdtj}tj ||kD||tjtjz}tjk7rtj|}|Sr[)rJr!rrPr\where)rr r^rirAr_negative_slopes rDr(z+create_epilogue_with_attr..inner_fns 'E # UEKK8<<5;;/DYY eUS\\.%++-V%VF #fe4MrUhardtanhrc |}tjk7r$tj|tj}tjtj |tj tjtj tj}tjk7rtj|}|Sr=)rJr!rrPrzr]r\)rr rirAr_ max_value min_values rDr(z+create_epilogue_with_attr..inner_fns 'E # UEKK8[[ E3<< 5;;#GH Y 4F #fe4MrU)rsubmulotherctt}dk7r|||dS|||Sr[)getattrr)ropr dims_diffr_ other_loaders rDr(z+create_epilogue_with_attr..inner_fnsKd#BA~,u-|E)*.inner_fnsSu%D 'EqydEKK84?%GMMrUz Unsupported epilogue attribute: )devicerAr(ranges) make_loaderrrr ValueErrorr Pointwise get_device) input_bufferrrrr(rnum_input_dimsnum_other_dimsrrrrArr_rrrrs ` @@@@@@@@@@rDcreate_epilogue_with_attrr0sf++-L  " " $E v~ , f$$$ + & ( +&&0 00 (     4  1   5 &    F"""6)$%*** *1-   F"""6)$%***9%a( 9%a(    & &&   w\2245U^^-."^3 ((*  D D  &   &   f~ww'')  ;D6BCC <<&&($$&  rUctd|Dr|}nt|ddrYtd|DsJtd|DsJ|Dcgc]%}|jjdj'}}nNtd|DsJtd|DsJ|Dcgc]}|jdj}}|J|Scc}wcc}w)Nc3<K|]}t|tywr=)r?rrBrs rDrEz!_get_loop_body..s 6:b( # 6sr original_fnc34K|]}t|dyw)rN)rnrs rDrEz!_get_loop_body..sDbwr=1Dsc3~K|]5}t|jjdjt7ywrN)r?rrL_bodyrrs rDrEz!_get_loop_body..s1GI 2>>..q177Bs;=c3PK|]}t|tj ywr=)r?rHrXrs rDrEz!_get_loop_body..sKRz"i&7&78Ks$&c3jK|]+}t|jdjt-ywr)r?rLrrrs rDrEz!_get_loop_body..s%P"z"''!*"2"2H=Ps13)rGrnrrLr)fn_list loop_bodiesrs rD_get_loop_bodyrs 6g 66 71:} -DGDD DDMT CJJB2>>..q177JKJK7KK KKPPP PP6=>2771:++>K>  "" "  K?s *C* Ccttj}|D]}|jjgt |j jDcgc]}|jc}z}|D]Y}|jD]H}|jdk7r|j|jtjjJ[|Scc}w)N call_method)r rJrA root_blockrrY subblocksvaluesr%rrr2r rb)rdtypes loop_bodybodygraphsrr^s rD_get_dtype_from_loopbodiesrs  $ &F E &&,,-#' (;(;(B(B(D#E1 DJJ1   EE  E77m+ 499%8%<%<=CCD E E E M1 sCtemplate epiloguescBdtjdttdttj fddt tj dttdtttffddt td ttjdtttfffd }|jsJ|j}|Dcgc]/}|jD]}|j |j1}}}|||Scc}}w) N epilogue_nodetemplate_buf_namesr_cx|jDcgc]}|j|vr |jc}Scc}wr=) get_readsrqr)rrreads rD!_get_indexes_of_template_buf_readzStemplate_fusion_with_epilogues_supported.._get_indexes_of_template_buf_reads> &//1 yy.. JJ   s!7index_of_template_buf_readepilogue_writesctt|}|dkDrd}d}||fS|dk(rd}d}||fS|dk(r|dtfd|D}|}||fStd)NrFrTc3<K|]}|jk(ywr=r)rBwriteiotbrs rDrEzftemplate_fusion_with_epilogues_supported.._check_supported_and_same_indexes..sOeU[[E1OszShould not reach here)rr rGAssertionError)rr num_indexes same_index supportedrs @rD!_check_supported_and_same_indexeszStemplate_fusion_with_epilogues_supported.._check_supported_and_same_indexess*%?@A ?JI*$$A JI*$$A .q1EOOOJ#I*$$!!89 9rUtemplate_outputsepilogue_nodescn|Dcgc]}|j}}|Dcgc] } || }}|Dcgc]}|jj}}t||Dcgc]\}} ||} }}t| \} } t | t | fScc}wcc}wcc}wcc}}wr=)rget_read_writeswritesziprG)rrrrrindexes_of_template_buf_readsepilogue_nodes_writesreadsrresultsr same_indexesrrs rD_template_fusion_supportedzLtemplate_fusion_with_epilogues_supported.._template_fusion_supporteds5EEqajjlEE"0)  .m=O P) %) IW! 7DM ) ) + 2 2! ! "%-/D" v .eV <  #&w- <9~s<000!F) !  sB"B'!B,+B1)r OperationrYrr{rrr rrr,r is_template get_outputsrr^) rrrrepiloguerCrrrs @@rD(template_fusion_with_epilogues_supportedrs || 9=c ejj  %$,UZZ$8%#C% tTz %.1"?31EI",,EW1 tTz 1*    !! !++-" ##%  66  N &&6 GG s4D)prr-rHrsys collectionsrcollections.abcrtypingrrr unittest.mockrr{rJtorch._prims_commonr torch.utils._ordered_setr torch.utils._sympy.printersr _CppPrintertorch.utils._sympy.symbolr r torch.utils._sympy.value_rangesrr dependenciesrrrrrrutilsrrr virtualizedrrrcommonrrrrr rUfloat64float16int64r@int16rTuint64uint32uint16rSr,r- complex32 complex64 complex128r.r/r0r1rI DTYPE_TO_ATENDEVICE_TO_ATENstrided_mkldnnLAYOUT_TO_ATENplatform _IS_WINDOWSrr7rMr]fxNodercrhrtr@rrrrrrrrWrapperHandlerrrr!rr>rErQrWrrrrYrrgrUrDrs "$** 0/A:7 :NN** MM7 MM8 MM6 KK   KK   KK  JJ LL* LL* LL* KK JJ NNJ OO) OO*  ,!" #$ } , ,) . MM< MM= MM; KK   KK   KK  JJ  LL- LL- LL- KK LL- LL- JJ  NNO  OO'!" OO(#$ * - ) 1 1- 4      MM> MM= llg%  .*KL <8ehhmm8(;83"53  <92[92x%%  8 94 ::84 ::"#OA,,#OLJ@J@` ++   S !&9> 6.cL& AHAH,01B,CAH 4:AHrU