Vhh UddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddlm Z m Z ddlmZddl mZmZmZmZmZmZmZmZmZddlmZddlZddlZddlZddlmZddlm Z!dd l"m#Z#dd l$m%Z%dd l&m'Z(dd l)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/ddl0m1Z1m2Z2ddl3m4Z4ddl5m6Z6m7Z7ddl8m9Z9m:Z:m;Z;mZ>m?Z?m@Z@mAZAmBZBmCZCddlDmEZEmFZFmGZGmHZHmIZImJZJerWddlKmLZLmMZMmNZNddlOmPZPmQZQmRZRmSZSddlTmUZUddlVmWZWmXZXmYZYddlZm[Z[edZ\eeeXgeWfZ]e^e[Z_ee`ejfZbe`ZcejjefdZgejefZiddZjGddejZke=d Gd!d"ZlejGd#d$ZnejGd%d&ZoejGd'd(ZpejGd)d*ZqejGd+d,ZreeleneoeqepfZsiZtd-eud.<Gd/d0ZviZwd1eud2< d dd3ZxGd4d5e Zy dd6Zz dd7Z{dd8Z| d dd9Z}ejddd:Z dd;Z dd<Zdd=Zejejej ejiej ejejejejejejejejejej f Dcic]}||c}Zd>eud?< dd@ZGdAdBZGdCdDe(Z'GdEdFZe j*dGe j,HZddIZGdJdKe6eeFeZejGdLdMZedidNeej8dOdPQdReej8dSdTdUVdWeej8dXdYdZVd[eej8d\d]d^Vd_eej8d`dadbVdceej8dddedcfdgeej8dhdidjVdkeej8dldmdndkodpeej8dqdpQdreej8dsdrQdteej8dudvQdweej8dxdyQdzeej8d{d|d}dz~deej8dddfdeej8dddVdeej8ddQdeej8ddQdeej8dddVdeej8dddVdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8ddQdeej8d„dìQdeej8dńdƬQdeej8dȄdɬQdeej8d˄d̬Qdeej8d΄dϬQdeej8dфdҬQZdeud<ddՄZGdքde:ZGd؄de<ZGdڄdeZejGd܄dݫZGdބd߫ZeZGddZGddZede`ZedeeZereejTeHeeeedfffZGddeeefZGddZGddeeeZejGddZejdddZGddZGdde7Zycc}w() annotationsN)autoEnum)chain) AnyCallablecastClassVarGeneric NamedTupleOptional TYPE_CHECKINGUnion)TypeVar)ELEMENTWISE_TYPE_PROMOTION_KIND)_pytree) OrderedSet)int_oo) PythonPrinter)free_symbol_is_typesymbol_is_typeSymT) bound_sympy ValueRanges)configmetrics)DtypePropagationOpsHandler)BasicMathOpsMixinDefaultHandler) boolean_opsDeferredLineBasegenerate_assertIndentedBuffer ir_dataclass ScopedDict sympy_dotsympy_index_symbol sympy_subs triton_typeunique)ops OpsHandlerOpsValue ReductionType StoreModeV)IteratorMutableMappingSequence)Buffer ChoiceCaller FixedLayoutIRNodeLoopBody)BaseScheduling Scheduler SchedulerNodePythonWrapperCodegen_Tschedulecxtjtjrtj d|yy)NzData type propagation: %s) schedule_log isEnabledForloggingDEBUGdebug)msgs N/home/dcms/DCMS/lib/python3.12/site-packages/torch/_inductor/codegen/common.pydata_type_loggerrKPs*  /6<0c<eZdZdZdZdZeddZeddZy) WorkspaceZeroModerr>rc||k(s|tjk(r|S|tjk(r|Std|d|d)NzWorkspaceZeroMode.combine(, ))rN UNINITIALIZEDNotImplementedErrorabs rJcombinezWorkspaceZeroMode.combineZsK 6Q+999H !// /H!$>qe2aU!"LMMrLcF|rtjStjSN)rN ZERO_ON_CALLrR) zero_fills rJ from_boolzWorkspaceZeroMode.from_boolbs $11 1 ...rLN)rUrNrVrNreturnrN)r[boolr]rN) __name__ __module__ __qualname__rRrZZERO_PER_GRAPH staticmethodrWr\rLrJrNrNUs9MLNNN//rLrNT)frozenceZdZUdZded<ded<ded<ded <d Zded <ejZd ed <e dddZ e ddZ e ddZ e ddZ ddZeZd dZd!dZed!dZeZeZeZd"dZd"dZd#dZd$dZy)% WorkspaceArga2A temporary buffer used for a single kernel, then discarded. Not registered as a traditional buffer since there are no users, so it would be dead code eliminated. Args: nbytes: The size of the buffer in bytes. zero_fill: Whether the buffer should be initialized to zero. sympy.ExprcountrN zero_mode torch.devicedevicestr outer_namews_ptr inner_name torch.dtypedtypecP|ttjjSrY)nextr1graph workspace_id)prefixs rJ unique_namezWorkspaceArg.unique_name}s!$qww334566rLc|j|jk(xr4|j|jk(xr|j|jk(SrY)rprrrlrTs rJcan_joinzWorkspaceArg.can_joins@ LLALL ( XQWW-? XAHHPQPXPXDX rLct|j|jztj|j|j|j |j |j|jSN)rirjrrrlrprn) rgrirNrWrjrrrlrprnrTs rJjoinzWorkspaceArg.joinsS''AGG#'// Q[[I''88||||   rLc|j|jk(r2|j|jk(r|j|jk(sJtt j |j |j tj|j|j|j|j|j|jSr|) rrrlrprgsympyMaxrirNrWrjrnrTs rJmaximumzWorkspaceArg.maximums GGqww 188qxx#7ALLALL)rlrrsizestride)irr7rlrrri)rr7s rJ get_layoutzWorkspaceArg.get_layouts.$;;****3   rLc"|jSrY)rrs rJlayoutzWorkspaceArg.layouts  rLc|jgSrY)rirs rJget_sizezWorkspaceArg.get_sizes |rLc8tjjgSrY)rSOners rJ get_stridezWorkspaceArg.get_strides }rLc|jSrY)rnrs rJget_namezWorkspaceArg.get_names rLcgSrYrdrs rJget_inputs_that_alias_outputz)WorkspaceArg.get_inputs_that_alias_outputs rLN) workspace_)rwrmr]rm)rUrgrVrgr]r^)rUrgrVrgr]rg)r]rk)r]rq)r]r7)r]list[sympy.Expr]r]rm)r] list[str])r_r`ra__doc____annotations__rptorchuint8rrrcrxrzr}rrget_device_or_errorrrpropertyrget_output_specmaybe_get_output_specmaybe_get_layoutrrrrrdrLrJrgrgis    OJE;$77        % !!!O&!rLrgcpeZdZUded<ded<ded<ej j Zded<dZd ed <y) TensorArgrmnamebufferrqrrrhoffsetN Optional[str]alias_of) r_r`rarrrZerorrrdrLrJrrs. I K FJ%"Hm"rLrc4eZdZUded<ded<eddZy)SizeArgrmrrhexprcyrYrdrs rJrzSizeArg.alias_ofsrLNr]r)r_r`rarrrrdrLrJrrs I  rLrceZdZUded<y) ConstexprArgrmrNr_r`rarrdrLrJrr IrLrceZdZUded<y)TMADescriptorArgrmrNrrdrLrJrrrrLrc0eZdZUded<ded<dZded<y) DeviceCodegenSchedulingConstructor schedulingWrapperConstructorwrapper_codegenNOptional[WrapperConstructor]cpp_wrapper_codegen)r_r`rarrrdrLrJrrs%%''8<5) CppScheduling) CppWrapperCpu)CppWrapperCpuArrayRef) CppWrapperGpu)CUDACombinedScheduling)HalideScheduling)MetalScheduling)TritonSchedulingr?cpu)cpphalidetritonc6tj|SrY)r cpu_backend)r cpu_backendss rJz+init_backend_registration..s?|F,>,>? KrLcuda)rrc6tj|SrY)r cuda_backend)r cuda_backendss rJr#z+init_backend_registration..sA}V-@-@A*MrLxpumps privateuseoner)_get_custom_mod_func Schedulingr@CppWrapperCodegen)rrcpp_wrapper_cpurcpp_wrapper_cpu_array_refrcpp_wrapper_gpurcuda_combined_schedulingrrrr)rrrwrapperr@rrr aot_inductorallow_stack_allocationr_C_get_privateuse1_backend_name torch.utils.backend_registrationr+ RuntimeError)rrrrrrrrr@private_backendr+rrrr"r's @@rJrrsm".@.@($(- '/ && $  K ""99 "  !(0-&  $  M   !'/#     !'/#     hh<<>O?* %o 6 >I  4\ B 23IJO"67J"K  _9L+#%#' :M_  ? +    s?,D11 D=<D=cLddlm}g|t||j|S)Nr)FlexibleLayout)rr;r'contiguous_strides)index index_varssizesr;s rJindex_prevent_reorderingr@s, $ UU TIj.*K*KE*RS TTrLc|t|<yrY)r)rldevice_op_overridess rJregister_device_op_overridesrCs(;V$rLclt|tsJtsddlm}m}ddlm}ddlm}t|S)Nr>)cpu_device_op_overridesmps_device_op_overrides)rB) rrmrrErFr$rBr()rlrErFrBxpu_op_overridess rJget_device_op_overridesrIs) fc "" " #F-@ #F ++rLzdict[torch.dtype, torch.dtype]DTYPE_TO_COMPUTATION_DTYPEcr|tvrtjS|dvrd|vr|dS|dS|dvrtjS|dvrtjS|dk(rd|vr|dS|dS|dk(rd|vr|dS|dS|d vr$|d}t j j|S|d k(rd|vr|dS|d Sy ) zK Given op name and a list of input dtypes, deduce the output dtype )to_dtype index_exprrr)randrandn) get_index randint64 load_seed reductionr>constant)loadstorestore_reductionto_dtype_bitcastN)r!rr^floatint64r1rur)op_nameargskwargsbuf_names rJdeduce_output_dtype_by_nameras +-zz  #*V"3vgAbA  {{   {{ K ")V"3vg@a@ J ")V"3vgAbA   7ww  ** & &")V"3vgAbA rLc`eZdZd dZd dZd dZd dZd dZddZe ddZ e ddZ y )DataTypePropagationc||_d|jji|_|jj D]\}}|j|j|< yNroot)body root_blockrugraphs subblocksitems)rrgkvs rJ__init__zDataTypePropagation.__init__-sU DOO))B  NN((* %DAqWWDKKN %rLc|j}|Dcgc]9}t|tjjs(|j dk7s8|;}}t |dk(rytd|D}|sytjtj|Dcgc])}|jtjj+c}Scc}wcc}w)N placeholderrc3K|]K}tj|jvxr)|jtjjduMywrY)OptimizationContextkeymetarr).0ns rJ zBDataTypePropagation.deduce_node_dtype_by_inputs..=sS)  # #qvv - B*../55TA B) sAA)all_input_nodesrrfxNodeoplenall functoolsreduce promote_typesrtrrrsrr)rnodeinputsrv input_nodesall_input_nodes_propagateds rJdeduce_node_dtype_by_inputsz/DataTypePropagation.deduce_node_dtype_by_inputs5s%% Auxx}}!=!$$-BWA   { q %() !) & " *   rmasked_subblock)r{rr|r^operatorgetitemdeduce_node_dtyperrm startswithrrar_r)rr output_dtypes rJrz%DataTypePropagation.deduce_node_dtypePs 77m # ;;( "s499~': ;;(** *))$))A,7 7$++s+++ ;; ! !"3 455d; ;8 ++ L    //55rLcn|jsJd}|jD]}tj|jvr|jtj}n t}|j ||_||jtj<|j dk(s|j }|S)Nr)nodesrrrsrtrrrr)rru graph_dtyperopt_ctxs rJrz#DataTypePropagation.propagate_graphks{{{-1 KK ,D"&&$))3))$7$;$;<-/ 2248GM18DII)-- .{{h&%mm  ,rLc>|j|jdSre)rrirs rJ propagatezDataTypePropagation.propagate}s##DKK$788rLc.||jSrY)r)clsrgs rJpropagate_loopbodyz&DataTypePropagation.propagate_loopbodys4y""$$rLcddlm}ddlm}t ||sJt |j |sJt j|j S)Nrr9)r=) loop_bodyr: schedulerr=r_bodyrcr)rrr:r=s rJpropagate_scheduler_nodez,DataTypePropagation.propagate_scheduler_nodesA(-$ ...$**h///"55djjAArLN)rgr:r]None)r torch.fx.Noder]Optional[torch.dtype])rrr]rq)ruztorch.fx.Graphr]r)r]r)rgr:r]r)rr=r]r) r_r`rarnrrrrr classmethodrrrdrLrJrcrc,sJ% * 66$9%%BBrLrcc6eZdZddd dfdZxZS)rT)simplifypc|r]t|tjrCttj dr)tj j j|}t|%|S)Nsizevars) rrExprhasattrr1rurrsuperdoprint)rrrr __class__s rJrzPythonPrinter.doprintsK  44*9U77##,,T2Dwt$$rL)rrhrr^rr^r]rm)r_r`rar __classcell__rs@rJrrs248D%%-1%=A% %%rLrcTeZdZdZeddZeddZeddZeddZeddZ eddZ eddZ edd Z edd Z edd Zedd Zedd ZeddZeddZeddZeddZeddZeddZy)OpDecompositionsz! Decomposes inductor ops c|SrYrd)values rJidentityzOpDecompositions.identitys  rLcrtjtjdtj|SNr>)r,truedivrUrint32xs rJ reciprocalzOpDecompositions.reciprocals"{{3<<5;;7;;rLc.tj||SrY)r,mulrs rJsquarezOpDecompositions.squareswwq!}rLctjtjdtjtj |Sr)r,subrUrfloat32erfrs rJerfczOpDecompositions.erfcs*wws||Au}}5swwqzBBrLctjtjtj|tj|SrY)r,rexprrrs rJerfcxzOpDecompositions.erfcxs,wwswwszz!}-sxx{;;rLctjtj|tjdtj Sr)r,rrrUrrrs rJexpm1zOpDecompositions.expm1s*wwswwqz3<<5==#ABBrLc tjtj|tjdt jdz t j S)Nr> r,rlogrUmathrrrs rJlog10zOpDecompositions.log10s7wwswwqz3<<DHHRL0@%--#PQQrLc tjtj|tjdt jdz t j S)Nr>rrrs rJlog2zOpDecompositions.log2s6wwswwqz3<<DHHQK#OPPrLc tjtj|tjt j dt jS)Nr)r,rrrUrrrrrs rJexp2zOpDecompositions.exp2s3wwswwq#,,txx{EMM"JKLLrLc tjtj|tjdtj Sr)r,raddrUrrrs rJlog1pzOpDecompositions.log1ps+wwswwq#,,q%++">?@@rLc tjdtj}tj|tj |tj tj|Sr)r,rUrrrrrnegrones rJsigmoidzOpDecompositions.sigmoidsCll1ekk*{{3SWWSWWQZ-@ ABBrLcrtj|tjdtjSNr)r,rrUrrrs rJreluzOpDecompositions.relus"{{1cll1ekk:;;rLcVtjtj|||SrY)r,rrryzs rJfmazOpDecompositions.fmaswwswwq!}a((rLcTtjtj||SrY)r,rLfloorrUrrs rJ floor_to_intzOpDecompositions.floor_to_int||CIIaL%00rLcTtjtj||SrY)r,rLceilrs rJ ceil_to_intzOpDecompositions.ceil_to_ints||CHHQK//rLcTtjtj||SrY)r,rLtruncrs rJ trunc_to_intzOpDecompositions.trunc_to_intrrLc tj||}tjtj|tjdt j tjtj|tj|}tj|tj|||Sr) r,modand_nerUrrsignbitwherer)rUrVrconds rJ remainderzOpDecompositions.remaindersy GGAqMxx FF1cll1ekk2 3 FF3;;q>3;;q> 2 yyswwq!}a00rLcTtjtj||SrY)r,rLroundrs rJ round_to_intzOpDecompositions.round_to_intrrLN)rOpVarTr]rrrr]r)rrrrrrr]r)rUrrrrqr]rrUrrVrr]r)r_r`rarrcrrrrrrrrrrrrrrrrrrrdrLrJrrs}<<CC<<CCRRQQMMAACC<<))1100111111rLrz[a-z0-9_.]+|\([^)]*\)|)flagsc|ddk7st|dkryd}t|ddD]3\}}|dk(r|dz }n |dk(r|dz}|dk(s!|t|dz k7s3y|dk(sJy)Nr(rFr>rQT)r| enumerate)stringriichars rJ_all_in_parensrs ayC3v;? EVABZ(4 3; QJE S[ QJE A:!s6{Q.  A:: rLceZdZed'dZed(dZed)dZed)dZed)dZed)dZ ed)dZ ed)dZ ed)d Z ed)d Z ed*d Zed+d Zed+d Zed+dZed+dZed+dZed,dZed-dZ d. d/dZ d0dZd1dZ d2 d3dZd4dZ d5dZ d6dZ d7dZ d8 d9dZd:dZde jBddd  d;d!Z"dd$Z%e&d?d%Z'e&d@d&Z(y)A OpOverridescrt|ts tj|s t |r|Sd|dSNrrQ)r CSEVariable_RE_PAREN_NOT_NEEDED fullmatchr)rs rJparenzOpOverrides.parens9 v{ +#--f5f%M6(!}rLct|SrY)repr)rrrs rJrUzOpOverrides.constant s E{rLc tjdtj}tj|tj |tj tj|Sr)r,rUrrrr libdevice_exprrs rJlibdevice_sigmoidzOpOverrides.libdevice_sigmoidsEll1ekk*{{3S->->swwqz-J KLLrLc,tj|SrY)r,absrs rJ libdevice_abszOpOverrides.libdevice_abswwqzrLc,tj|SrY)r,sqrtrs rJlibdevice_sqrtzOpOverrides.libdevice_sqrtsxx{rLc,tj|SrY)r,cosrs rJ libdevice_coszOpOverrides.libdevice_cosrrLc,tj|SrY)r,sinrs rJ libdevice_sinzOpOverrides.libdevice_sin!rrLc,tj|SrY)r,rrs rJ libdevice_logzOpOverrides.libdevice_log%rrLc,tj|SrY)r,rrs rJrzOpOverrides.libdevice_exp)rrLc2dtj|S)N~rr rs rJ bitwise_notzOpOverrides.bitwise_not-s;$$Q'())rLc2tj|dS)Nz == 0r%)rUs rJ logical_notzOpOverrides.logical_not1s##A&'u--rLc\tj|dtj|S)Nz & r%rrs rJ bitwise_andzOpOverrides.bitwise_and5+##A&'s;+<+> r%r*s rJbitwise_right_shiftzOpOverrides.bitwise_right_shiftEr3rLc.tj||SrY)r,rrTs rJ int_truedivzOpOverrides.int_truedivIs {{1a  rLcTtj|tj|SrY)r,rVrInteger)rrs rJrSzOpOverrides.load_seedPsxxemmF344rLTc*tt|SrY)r(rm)rvarrcheckwrap_negs rJindirect_indexingzOpOverrides.indirect_indexingTs"#c(++rLcDtt|jd)Nz,: check_bounds should be handled by CSEProxyrSrr_rrrloweruppers rJ check_boundszOpOverrides.check_bounds]s'"Dz""##O P  rLcDtt|jd)Nz$: load should be handled by CSEProxyr@rrr=s rJrVzOpOverrides.loadds%!Dz""##G H  rLNcDtt|jd)Nz%: store should be handled by CSEProxyr@rrr=rmodes rJrWzOpOverrides.storeis'"Dz""##H I  rLcDtt|jd)Nz/: store_reduction should be handled by CSEProxyr@rrr=rs rJrXzOpOverrides.store_reductionps%!Dz""##R S  rLcDtt|jd)Nz): reduction should be handled by CSEProxyr@rrr src_dtypereduction_typers rJrTzOpOverrides.reductionus'"Dz""##L M  rLcDtt|jd)Nz$: scan should be handled by CSEProxyr@rdtypes combine_fnvaluess rJscanzOpOverrides.scans'"Dz""##G H  rLcDtt|jd)Nz$: sort should be handled by CSEProxyr@rrRrTstable descendings rJsortzOpOverrides.sorts'"Dz""##G H  rLcDtt|jd)Nz): bucketize should be handled by CSEProxyr@rrT boundariesboundary_indicesindexing_dtyperightsortersorter_indicess rJ bucketizezOpOverrides.bucketizes'"Dz""##L M  rLcDtt|jd)Nz2: halide_clamp only implemented for Halide backendr@)rrrr<s rJ halide_clampzOpOverrides.halide_clamps%!Dz""##U V  rLr>) constraintsrris_purepackcDtt|jd)Nz<: inline_asm_elementwise only implemented for Triton backendr@)rasmrfrrrgrhrs rJinline_asm_elementwisez"OpOverrides.inline_asm_elementwises'"Dz""##_ `  rLcDtt|jd)Nz.: ops.output should not appear at codegen timeAssertionErrorrr_)rr^s rJrzOpOverrides.outputs%Dz""##Q R  rLcDtt|jd)Nz3: ops.placeholder should not appear at codegen timermrr=s rJrpzOpOverrides.placeholders%Dz""##V W  rLc0dfd }|_d|_|S)NcJtt|jd)Nz does not implement ops.r@)rr^r_rs rJ unimplementedz1OpOverrides._unimplemented..unimplementeds*%:&&''?vF rLT)rrr^rr_rr]r)r_is_unimplemented)rrss` rJ_unimplementedzOpOverrides._unimplementeds   "& )- &rLcpt||d}tt|d}| xs||k(xs t|ddS)NrtF)getattrr-)rrfn default_fns rJ_is_unimplementedzOpOverrides._is_unimplementeds? S$ %Zt4 vSz)SWR9KU-SSrLcP|dvsJ|tjD]\}}t||}|/|j|s&t |||j |C||j vsJd|d|j||_t ||t|y)N)rrcppvecrr)zmultiple definitions of z on ) pointwise_overrides_datarkrwrzsetattrru__dict__r_rc)rrfuncnamedataimpls rJ_initialize_pointwise_overridesz+OpOverrides._initialize_pointwise_overridessEEMvME6<<> ;NHd4(D|((2C3+=+=h+GHs||3.xjS\\NK3!) X|D'9: ;rL)rrr]r)rzUnion[bool, float, int]rrrqr]rr)rUrr]r)rrrrr]rr)rrmrrr]rTT) r;rrUnion[sympy.Expr, int]r<r^r=r^r] sympy.Symbol rrhrrhrBr^rCr^r]r)rrmr=rhr]rrY) rrmr=rhrrrIr0r]r)rrmr=rhrrr]r) rrrqrNrqrOr/r!Union[OpVarT, tuple[OpVarT, ...]]r]r)rRtuple[torch.dtype, ...]rSzFCallable[[tuple[OpVarT, ...], tuple[OpVarT, ...]], tuple[OpVarT, ...]]rTtuple[OpVarT, ...]r]r) rRrrTrrXr^rYr^r]rNN)rTrr].tuple[str, sympy.Expr, sympy.Expr, sympy.Expr]r^rr_rqr`r^ra Optional[tuple[str, sympy.Expr]]rbzOptional[OpVarT]r]r)rrrrhr<r^r]r)rrrjrmrfrrrrqrgr^rhrr]r)r^rr]r)r=rr]r)rrmr]zCallable[..., OpVarT]rrmr]r^)rrmr]r))r_r`rarcr rUrrrrrr!rr&r(r+r.r0r2r5r7rSr>rDrVrWrXrTrUrZrcrerrrkrrprurrzrrdrLrJrrsMM**..BBBBBBCCCC!! 55 , ,%, ,  ,  ,  &0 9= FJ   NR  * 39 AJ        &  1  +    '     #       '  #        $48+/    C  !  $    1  )      &*"]]      #             TT  ; ;rLrc|eZdZUded<ded<dZded<dZded<ejZd ed <dZ ded <dZ ded <y) OverridesDatarmrzCallable[..., str]rNzOptional[Callable[..., str]]rr|rtype_promotion_kindrr)) r_r`rarrr|rDEFAULTrrr)rdrLrJrrsQ I +/F (/+/F (/'//8,0F (/(,C %,rLrairy_aicd|dS)Nzairy_ai_forward(rQrdrs rJr#r#s(1-rLspecial_airy_ai)rrr bessel_j0cd|dS)Nzbessel_j0_forward(rQrdrs rJr#r#*1#Q/rLcd|dS)Nz libdevice.j0(rQrdrs rJr#r#=1-rLspecial_bessel_j0)rrrr bessel_j1cd|dS)Nzbessel_j1_forward(rQrdrs rJr#r#rrLcd|dS)Nz libdevice.j1(rQrdrs rJr#r#rrLspecial_bessel_j1 bessel_y0cd|dS)Nzbessel_y0_forward(rQrdrs rJr#r# rrLcd|dS)Nz libdevice.y0(rQrdrs rJr#r# rrLspecial_bessel_y0 bessel_y1cd|dS)Nzbessel_y1_forward(rQrdrs rJr#r#rrLcd|dS)Nz libdevice.y1(rQrdrs rJr#r#rrLspecial_bessel_y1digammacd|dS)Nz calc_digamma(rQrdrs rJr#r#s aS*rLc |dS)Nz .digamma()rdrs rJr#r#s A3j)rL)rrr|rrcd|dS)Nz calc_erfcx(rQrdrs rJr#r# A3a(rLcd|dS)Nzlibdevice.erfcx(rQrdrs rJr#r#s+A3a0rL special_erfcxrcd|d|d|dS)Nz std::fma(rPrQrdrs rJr#r##sis"QCr!A6rLcd|d|d|dS)Nzfmadd(rPrQrdrs rJr#r#$ss"QCr!A6rLcd|d|d|dS)Nzlibdevice.fma(rPrQrdrs rJr#r#%ss"QCr!A>rL)rrr|rrigammacd|d|dSNz calc_igamma(rPrQrdr*s rJr#r#+<s"QCq1rLigammaccd|d|dSNz calc_igammac(rPrQrdr*s rJr#r#0=2aS2rLgammainccd|d|dSrrdr*s rJr#r#5rrLspecial_gammainc gammaincccd|d|dSrrdr*s rJr#r#:rrLspecial_gammaincci0cd|dS)Nzcalc_i0(rQrdrs rJr#r#?1orLcd|dSNzlibdevice.cyl_bessel_i0(rQrdrs rJr#r#@3A3a8rLc |dS)Nz.i0()rdrs rJr#r#As A3erL)rrrr|ri0ecd|dS)Nz calc_i0e(rQrdrs rJr#r#F !A&rLc |dS)Nz.i0e()rdrs rJr#r#Gs A3frL special_i0ei1cd|dS)Nzcalc_i1(rQrdrs rJr#r#LrrLcd|dSNzlibdevice.cyl_bessel_i1(rQrdrs rJr#r#MrrL special_i1i1ecd|dS)Nz calc_i1e(rQrdrs rJr#r#RrrL special_i1elog_ndtrcd|dS)Nzcalc_log_ndtr(rQrdrs rJr#r#Wsqc+rLspecial_log_ndtrmodified_bessel_i0cd|dS)Nzmodified_bessel_i0_forward(rQrdrs rJr#r#]3A3a8rLcd|dSrrdrs rJr#r#^rrLspecial_modified_bessel_i0modified_bessel_i1cd|dS)Nzmodified_bessel_i1_forward(rQrdrs rJr#r#crrLcd|dSrrdrs rJr#r#drrLspecial_modified_bessel_i1modified_bessel_k0cd|dS)Nzmodified_bessel_k0_forward(rQrdrs rJr#r#irrLspecial_modified_bessel_k0modified_bessel_k1cd|dS)Nzmodified_bessel_k1_forward(rQrdrs rJr#r#nrrLspecial_modified_bessel_k1ndtrcd|dS)Nz calc_ndtr(rQrdrs rJr#r#ts 1#Q'rL special_ndtrndtricd|dS)Nz calc_ndtri(rQrdrs rJr#r#yrrL special_ndtri polygammac|d|d|d|dS)Nz == 0 ? calc_digamma(z) : calc_polygamma(rPrQrdr*s rJr#r#~s"A33A36I!BqcQRSrLscaled_modified_bessel_k0cd|dS)Nz"scaled_modified_bessel_k0_forward(rQrdrs rJr#r#:1#Q?rL!special_scaled_modified_bessel_k0scaled_modified_bessel_k1cd|dS)Nz"scaled_modified_bessel_k1_forward(rQrdrs rJr#r#rrL!special_scaled_modified_bessel_k1spherical_bessel_j0cd|dS)Nzspherical_bessel_j0_forward(rQrdrs rJr#r#s4QCq9rLspecial_spherical_bessel_j0zetacd|d|dS)Nzzeta(rPrQrdr*s rJr#r#s52aS*rL special_zetachebyshev_polynomial_tcd|d|dS)Nzchebyshev_polynomial_t_forward(rPrQrdr*s rJr#r#:1#Rs!DrLspecial_chebyshev_polynomial_tchebyshev_polynomial_ucd|d|dS)Nzchebyshev_polynomial_u_forward(rPrQrdr*s rJr#r#rrLspecial_chebyshev_polynomial_uchebyshev_polynomial_vcd|d|dS)Nzchebyshev_polynomial_v_forward(rPrQrdr*s rJr#r#rrLspecial_chebyshev_polynomial_vchebyshev_polynomial_wcd|d|dS)Nzchebyshev_polynomial_w_forward(rPrQrdr*s rJr#r#rrLspecial_chebyshev_polynomial_wlegendre_polynomial_pcd|d|dS)Nzlegendre_polynomial_p_forward(rPrQrdr*s rJr#r#9!BqcCrLspecial_legendre_polynomial_pshifted_chebyshev_polynomial_tcd|d|dS)Nz'shifted_chebyshev_polynomial_t_forward(rPrQrdr*s rJr#r#B1#Rs!LrL&special_shifted_chebyshev_polynomial_tshifted_chebyshev_polynomial_ucd|d|dS)Nz'shifted_chebyshev_polynomial_u_forward(rPrQrdr*s rJr#r#rrL&special_shifted_chebyshev_polynomial_ushifted_chebyshev_polynomial_vcd|d|dS)Nz'shifted_chebyshev_polynomial_v_forward(rPrQrdr*s rJr#r#rrL&special_shifted_chebyshev_polynomial_vshifted_chebyshev_polynomial_wcd|d|dS)Nz'shifted_chebyshev_polynomial_w_forward(rPrQrdr*s rJr#r#rrL&special_shifted_chebyshev_polynomial_whermite_polynomial_hcd|d|dS)Nzhermite_polynomial_h_forward(rPrQrdr*s rJr#r#s82aSBrLspecial_hermite_polynomial_hhermite_polynomial_hecd|d|dS)Nzhermite_polynomial_he_forward(rPrQrdr*s rJr#r#r rLspecial_hermite_polynomial_helaguerre_polynomial_lcd|d|dS)Nzlaguerre_polynomial_l_forward(rPrQrdr*s rJr#r#r rLspecial_laguerre_polynomial_lzdict[str, OverridesData]r}ctfdtjjtjjtjj tjj fDS)Nc3&K|]}|v ywrYrd)rurrs rJrwz$is_buffer_removed..s   )anyr1ruremoved_bufferskernelinplaced_to_removers`rJis_buffer_removedr*sU  GG # # HH $ $ GG & & HH ' '   rLc4eZdZdZdfd ZddZddZxZS) DeferredLinezHA line that can be 'unwritten' by adding name to V.graph.removed_bufferscVt||||_t|trJyrY)rrnrrr")rrliners rJrnzDeferredLine.__init__s+  d$45555rLcFt|js |jSyrY)r*rr.rs rJ__call__zDeferredLine.__call__s +99 rLc.t|j|SrY)r,r)rr.s rJ _new_linezDeferredLine._new_linesDIIt,,rL)rrmr.rmr)r.rmr]r,)r_r`rarrnr0r2rrs@rJr,r,sR6  -rLr,ceZdZdddZy) BracesBuffercHtjdfd }|S)Nc3KtD](}jdxjdz c_*t D](}xjdzc_jd*dt D](}jdxjdz c_*tD](}xjdzc_jd*yw)N{r>})range writeline_indent)_rrs rJctxz BracesBuffer.indent..ctxs6] "s# !  "F7^ $ ! s# $ F7^ "s# !  "6] $ ! s# $sC C#)r]Iterator[None]) contextlibcontextmanager)rrr=s`` rJindentzBracesBuffer.indents$  " " $ # $u rLN)r>)rrr]z'contextlib.AbstractContextManager[None])r_r`rarArdrLrJr4r4srLr4c"eZdZUded<ded<y)InplacedBufferrmrpr other_namesNrrdrLrJrCrC s OrLrCc.eZdZUded<dZded<ddZy) ArgNamermrFr^ is_constexprcB|j|jrdSdS)Nz : tl.constexprrG)rrGrs rJ full_namezArgName.full_names*))$2C2C.LMMLMMrLNr)r_r`rarrGrIrdrLrJrFrFs IL$NrLrFceZdZddZy) RemovedArgcy)NREMOVEDrdrs rJ__str__zRemovedArg.__str__srLNr)r_r`rarNrdrLrJrKrKsrLrKceZdZe ddZddZddZeddZddZddZ ddZ ddZ dd Z dd Z dd Zd d Zd!d Zd"dZd#dZd$dZ d%dZd&dZd'dZd(dZy)) KernelArgsc~|j|t}t|tr|t |x||<}|S|SrY)getrMrrKr|)rwodictrresult new_results rJ_lookupzKernelArgs._lookup!sD */4)A fj )*0#e*'> >E$K*  rLcJi|_i|_i|_i|_g|_yrY) input_buffersoutput_buffersinplace_buffersrworkspace_argsrs rJrnzKernelArgs.__init__-s)-/ACMO/1 24rLc djdjtt|j|j |j |jgS)NzKernelArgs({})rP)formatr}maprrXrYrZrrs rJ__repr__zKernelArgs.__repr__4sS&& II**++,,    rLc"t|tSrY)rrKr)s rJ_buffer_is_marked_removedz$KernelArgs._buffer_is_marked_removedCs$ ++rLc:tjjr4tjjjj ||}|tjj vsJ|||j vrtt|j |S||jvr'tt|j|jS|jdr|jd|j|S|jd|j|S)Nseedin_ptr)r1rurmutation_real_namerRr&rYr rmrZrCrprrVrXrs rJinputzKernelArgs.inputHs 77  77$$77;;D$GD1772228D82 4&& &T0067 7 4'' '(<(R!R  /01[)C03D  ,03D  -! s D)>D.c@t|tj|tjj tj }t|jD]\}}tj||r?|j}tj|||j|<|j|fcS|j|jk7r|j|jk7rJ|jj||jdfS)a Allocate or extend a workspace buffer of nbytes bytes. This function manages the allocation of a workspace buffer. It either creates a new WorkspaceArg or extends an existing one. Note: - Calling this function will in-place mutate the args by adding or updating a WorkspaceArg. - The codegen for generating the Python argdefs and call_defs will check this field and allocate the buffer accordingly. - A new argument "ws_ptr" will be present in the generated code. Args: nbytes (sympy.Expr): The number of bytes to allocate. zero_fill (bool): Whether to initialize the buffer to zero. Returns: Tuple[str, int]: A tuple containing: - "ws_ptr": A string identifier for the workspace pointer. - offset: An integer representing the byte offset in the workspace. )rirjrlrnr)rgrNr\r1ruget_current_device_or_throwrxrr[rzrir}rprnrk)rnbytesr[argr existing_argrs rJ workspacezKernelArgs.workspacevs.'11)<77668#//1   ))<)<= OA|$$\37%++)5):):<)M##A&#..66''3>>9 ++s~~= >   ""3'~~q  rLc ~tjj}t|tj t jdd|jd|j|}|jD]#}|j|jk(s||k(r#J|jj||jS)a Lazily allocate a graph-wide semaphores buffer with at least min_size. This is a single buffer shared by all kernels and zero initialized once at graph start. Each kernel must leave the buffer zeroed on exit. Warning: multiple calls to this function will return the same buffer. Args: min_size: the number of int32 semaphores required Returns: name of the semaphores buffer sem_ptr semaphores_r<)rirjrrrprnrl) r1rurtrgrNrbruint32rr=r[rprk)rmin_sizecurrent_devicervrws rJ semaphoreszKernelArgs.semaphoress<<>'66,, $^%8%8$9>;O;O:PQ!  !// +L&&#..8l*** + ""3'~~rLcft|tsJt||ftj|}||j vr|j |S|j j vr0tfd|j j D|j |<S)Nc3FK|]}|jsdyw)r>N)r)rurmrs rJrwz)KernelArgs.seed_offset..sU1!,,tBTQUs!!)rrrrr9rrTsum)rrrs ` rJ seed_offsetzKernelArgs.seed_offsets%%;U U';;% e$ DMM !==' ' 4=='') )&U (<(<(>UUVW  $ e rLct|tjsJt||f|jdk(rd|j |<y|j d|j |S)Nrcks)rrSymbolrrrrVrs rJrzKernelArgs.sizesX$ -AT D/AA- 99 "(DMM$ ||D$--66rLct|jj|jj|jjSrY)rrXkeysrYrrs rJ call_nameszKernelArgs.call_namessA    # # %t':':'?'?'A4==CUCUCW  rLc|jj|d}|t|ts |jS|j j|d}|t|ts|S|j j|dS)z; Returns inner name of a given outer name. N)rZrRrrKrprYrX)rrinplacedrms rJarg_namezKernelArgs.arg_names}''++D$7   8Z(H&& &))--dD9  ":k:+N !!%%dD11rLc|SrYrd)rrnrrs rJ wrap_ptr_argzKernelArgs.wrap_ptr_args rLct|SrY)rm)rrs rJ wrap_size_argzKernelArgs.wrap_size_args 4yrLcddlm}m}g}g}g}t|jj D]}t |tr|jd}|j}tjj|} || } |j| d||j|j|| |j| d|jj!D]\}}||jvrtjj|} || } |jd| d||j|j|| |jd| d|j"j!D]\}} ||jvst | tr%tjj|} || } |j| d| |j|j|| |j| d|j$j!D]\}}|jd|d||j|j'||jd|tjj(sltjj(j+||j,rJd|||fS) Nr>) DTYPE_TO_CPP INDEX_TYPErNz* *zconst  zWorkspace not supported on CPU ) cpp_utilsrrr+rZrTrrKrDrpr1rurrkrrXrkrYrr wrapper_codeensure_size_computedr[) rrr call_argsarg_defs arg_typesrouterinnerrr cpp_dtype maybe_inners rJ cpp_argdefszKernelArgs.cpp_argdefss7  t33::<= .H(J/((,E''EGG%%e,E$U+I OOykE73 4   T..ue< =    {!_ - .!..446 4LE5,,,GG%%e,E$U+I OOfYKr%9 :   T..ue< =   vi[2 3 4#'"5"5";";"= . E;,,, ; 0SGG%%e,E$U+I OOykK=9 :   T..ue< =    {!_ - .!MM//1 ALE5 OOfZL%9 :   T//6 7   vj\2 3ww##$$99%@  A &&I(II&I--rLc g}g}g}g}t|jjD]}t|tr|j t |j|j |jd|j tjj|jd|j t|j|jdtjj|jdt|jj|j jD]\}}||jvst|tr%|j t ||j ||j tjj||j t||tjj||j"jD]\}}|j t ||j ||j t%||j t'||tjj(stjj(j+||j,D]m}|j t |j|j |j.|j ||j |j0o||||fS)NrN)rrrr)r+rZrTrrKrkrFrprDr1rurrrrXrkrYrrrrrr[rnrr) rrrrprecompile_argsrrrrvs rJpython_argdefszKernelArgs.python_argdefss#%! ') /1t33::<= H(J/ OOGH$7$78 9   X11"5 6   QWW..x/C/CB/GH I  " "!,,#//3''++H,@,@,DE  "    $ $ &(;(;(A(A(C  LE5,,, 5*0M OOGEN +   U #   QWW..u5 6  " " ''++E2  !MM//1 ALE5 OOGEN +   U #   T%[ )  " "75%#8 9ww##$$99%@  A&& (C OOGCNN3 4   S^^ ,  " "3 '   SYY '  ( OY>>rLc#Kt|jjD]}t|tr|j D]}|t jjvs|t jjvr<||jvr|j||jf||jvsvtt|j||jfywrY)r+rZrTrrKrDr1rur(r'rXrprYr rm)rrothers rJaliaseszKernelArgs.aliases@st33::<= UH(J/!-- UQWW777 ; ;;D...,,U3X5H5HHHD///sD$7$7$>?ATATTT U Us B9C,<0C,ct|jj|ttxr.t|j j|ttSrY)rrYrRrMrKrZrs rJ is_removedzKernelArgs.is_removedOsK    # #D' 2J N--11$@*M NrLclt}t|jjD]1}t |t r|j |jd3|jjD]5\}}||jvst |t r%|j |7|S)NrN) rr+rZrTrrKrrDrYrk)r live_outsrrrs rJlive_output_bufferszKernelArgs.live_output_buffersWsL t33::<= 4H(J/ MM(..r2 3 4!//557 !LE5,,, 5*0M MM%  !rLN)rwrmrSz6Union[dict[_T, Union[str, RemovedArg]], dict[_T, str]]rrAr]rmr]rr)rrr]r^r)rlrmrmrmr]r)rurhr[r^r]ztuple[str, int])r}rhr]rm)rrmrrr]rm)rrr]rm)r]z Iterator[str])rrmr]r)rnrmrrrqr]rm)r SymbolLiker]rm)r]z&tuple[list[str], list[str], list[str]])r]z?tuple[list[ArgName], list[str], list[KernelArgType], list[Any]])r]zIterator[tuple[str, str]]r)r]zOrderedSet[str])r_r`rarcrVrnr_rarfrrrrxrrrrrrrrrrrrrdrLrJrPrP s  E    5  ,, @B44'!R8 7 2'.R/? H/?b UN rLrPcZeZdZdZ d d fd Zd dZd dZd dZd dZd dZ xZ S)r aDA CSEVariable is just a name for an expression but it is useful to be able to annotate them on a backend dependent basis. To do so, the backends can simply overload `Kernel.create_cse_var` The "CSEVariable.update_on_args" method gives you a hook for annotations See example of TritonCSEVariable in triton.py c~t|t|tsJ||_||_d|_||_yr)rrnrrrbounds use_countrr)rrrrrrs rJrnzCSEVariable.__init__ks< &+...   rLc|jSrYr)rs rJrNzCSEVariable.__str__xs yyrLc,t|jSrY)hashrrs rJ__hash__zCSEVariable.__hash__{sDIIrLcXt|txr|j|jk(SrY)rr r)rrs rJ__eq__zCSEVariable.__eq__~s!%-I%** 2IIrLcyrYrd)rrr^r_s rJupdate_on_argszCSEVariable.update_on_argss rLcN|jjd|jdSr )rr_rrs rJr_zCSEVariable.__repr__s$..))*!DII=::rLrY)rrmrValueRanges[Any]rrrr)r]r)robjectr]r^)rrmr^rr_rr]r) r_r`rarrnrNrrrr_rrs@rJr r dsH(,  ! % J ;rLr  AugmentedKeyT)defaultCSEVariableType)boundr.c>eZdZdZ d ddZddZddZddZddZddZ dd Z dd Z dd Z e jd d dd  ddZe jdf ddZe jdf ddZy)CSEz Common subexpression eliminationNc||_||_i|_||_|xsi|_|xsi|_|xst j|_t|_ |xsi|_ yrY) rwsuffix_cache name_prefix store_cachereduction_cache itertoolsriiter_buffer_idsrinvalidated_stores varname_map)rrwrr iter_buffersrrrs rJrnz CSE.__init__sm  FH &ALARPR  !r 6B5VY__EV3=<7B7HbrLc6g|jjD]2\}}||vs |j|=|jj|4|r9|jjDcic] \}}||vs ||c}}|_yi|_ycc}}wrY)rrkrrr)r keep_varsrtmprlrms rJ invalidatezCSE.invalidates44++1134 2ID#)#$$T*''++D1 2 ,0KK,=,=,?RDAq1 >1a4RDKDKSs 1 B>Bc t||j|j|j|j|j |j |jS)N)rwrrrrrr)rrwrrrrrrrs rJclonez CSE.clonesPtDz;;;;((--(((( 00  rLc|j}t|j|_t|j|_t|j|_|S)zNReturn a copy of using ScopedDict so changes to *_cache aren't visible in self)rr&rrr)rnew_cses rJ scoped_copyzCSE.scoped_copysH**,#DKK0",T-A-A"B()9)9:rLc"tt|S)z@Override this method to augment cache key with backend specifics)r rr cache_keys rJ augment_keyzCSE.augment_keysM9--rLc@||j|j|<yrYrr)rrros rJputzCSE.puts36 D$$Y/0rLc<|j||jvSrY)rrrs rJcontainsz CSE.containss *dkk99rLcX|jj|j|dSrY)rrRrrs rJtry_getz CSE.try_gets"{{t// :DAArLc>|j|j|SrYrrs rJrRzCSE.gets{{4++I677rLT)rwrite assignmentrrc t|tr |j}|s|sJt|trE|jj ||_|xj dz c_tt|St|tr|j}n1t|tr |j}nt|tsJ|}|j|}|s|j||}|j!|||rt"j$j&r+t"j$j&j)|dt|trP|r |j+|j,|d|j/||j+|j0|St|trM|sJ|j+|j3|j,|d|j|j0|S|r |j,|d||j0} n||j0} |j+| |rKt4j6j8r1|/d|j,|dt;|d} |j+| |S|jj ||_|xj dz c_|S) Nr>T) only_oncez =z = tl.static_assert( .dtype == rQ)rr.rr rtightenrr rr$getvaluer"r.rmrnewvarrr1r' current_nodecodegen_originating_infor:rwsplicerr2r test_configsruntime_triton_dtype_assertr*) rrrrrrrrrr;r. assert_lines rJgeneratez CSE.generatesi dH %::D "" dK (++--f5DK NNa N. . n - I . / IdC( ((Ill9%++fe,C HHY $88((HH))BB$CdN3!((DKK=R)@AMM$'$$T[[12 1 &67%%:$$$++se3tyyk$++'WX, %""&++se3tfT[[MJ"& }5$$T*#"//KK!-(9$++se:VabgVhUiij&k ((5  ++F3CJ MMQ M rLc|jt|j}tjj |||}||j |<|SrY)rrtrr1r'create_cse_varr)rrrrvar_namer;s rJrz CSE.newvarsR &&'T-A-A(B'CDhh%%h>%(" rLctj|jvfdtjj ||}||j<|S)NcdS)Nzduplicate name: rdr)srJr#zCSE.namedvar..0s4DTF2KrL)r _check_valuerr1r'r)rrrrrr;s ` rJnamedvarz CSE.namedvar)sS  (( (*K hh%%dFE:!$ rL)rGrGrNNNN)rwrmrrmrrmrzOptional[itertools.count[int]]rz.Optional[MutableMapping[str, CSEVariableType]]rz|jj|||yrY)r__exit__)rexc_typeexc_valexc_tbs rJrzCodeGen.__exit__@s   7F;rLrrrrrrrrr]r)r_r`rarnrrrrs@rJrr7s1r)rrnrgenerated_kernel_countrPr^r$loadscomputestoresnum_load num_reductionrr rcserrmmust_keep_buffersstore_buffer_names _load_mask _load_otherrnode_to_boundsr&r(inplace_update_buffersmin_elem_per_thread kernel_name)rr^increase_kernel_countrs rJrnzKernel.__init__Is   * *a / *(JL #% %' $&  .1$2D2Ddkk.R!+C!2",S/"3)-4859OS)#0",S/"3 79##$ *.rLc#K|j}||_|jjj|_ d||_y#||_wxYwwrY)rrr get_boundsr)rrpriors rJset_current_nodezKernel.set_current_nodeksO!! "jj//1<<> &  %D D sAAAA AAc#K||}|dux}r t}|j}|j}|j}|j}||_||_||_|j |_ d||_||_||_||_|r |rJdyy#||_||_||_||_|r |rJdwwxYww)Nz$unexpected store inside swap_buffers)r$rrrrr) rlbcbsbdisallow_storesrrrrs rJ swap_bufferszKernel.swap_buffersus :B Dj (? (!B ,,hh   ??$ F DJ"DL DKDHEEEv2 DJ"DL DKDHEEEv2sA/C 2B6)C *C  C ctrYrrFs rJrVz Kernel.loadrrLc|j} |j|_|j||||_S#||_wxYw)z+A load the depends on an index we have read)rrrV)rrr=r!s rJ indirect_loadzKernel.indirect_loads8  DJ99T5)DJDJs "8 ActrYrrKs rJrXzKernel.store_reductionrrLctrYrrHs rJrWz Kernel.store "!rLctrYrrMs rJrTzKernel.reduction "!rLctrYrrQs rJrUz Kernel.scans "!rLctrYrrWs rJrZz Kernel.sortr0rLctrYrrs rJ var_rangeszKernel.var_rangesrrLct)z3 See [Note: Inductor bucketize op] rr\s rJrczKernel.bucketizes "!rLctrYrrs rJassert_functionzKernel.assert_functionrrLc Zt|tr t|}t|tsJ|t|tsJ|t|tsJ|r|rd|d|d|d|d }|d|d|}n|r |d|}|}n |sJ|d|}|}|r d|d|d}|jd|d|dS) Nrz <= z) & (z < rQz) | ~(z, "index out of bounds: z"))rr rmr7)rr;rBrCmaskr cond_prints rJindirect_assertzKernel.indirect_asserts c; 'c(C#s###} 5# 666} 5# 666 UugT#eC5E7! ?  %%a&:&:4&@A rLcH|jt| |||yrY)remove_kernel_local_buffersrr)rrrrrs rJrzKernel.__exit__s  ((* 7F3rLctjjsytfd|jD}tt |jD]N}||j vs||jjvs+j||s>j|PD]}||jjvrw|jj|}t|trEtfd|jD}|r|j!||j"j||j%|y)z Any buffers that are both created and have a last use in the same kernel can be removed. Note that V.graph.scheduler can be None when codegening triton template kernels. Nc3tK|]/}|jvrj|j1ywrY) name_to_bufdefining_op_name)rurnrs rJrwz5Kernel.remove_kernel_local_buffers..s;& i+++  ! !# & 7 7 9& s58c3&K|]}|v ywrYrd)rurvnames_to_removes rJrwz5Kernel.remove_kernel_local_buffers..,sKaQ/1Kr$)r1rurrrrmrr^rX$can_buffer_be_removed_through_fusionrrZrrKr}rDremove_inplace_bufferr( remove_buffer)rfused_node_namesrrnremoverKrs @@rJrEz"Kernel.remove_kernel_local_buffers s*GG%%  %& ..&   %S/+++ *DD222 7 77BB* ##D) *$ )Dtyy000ii//5c:.K3??KK..t4''++D1""4( )rLctjd|t|jj|<|j j |y)Nzremove_buffer(%r))rrHrMr^rYr&rrs rJrNzKernel.remove_buffer3s; %t,)0   &   &rLctjd|t|jj|<|j j |y)Nzremoving_inplace_buffer(%r))rrHrMr^rZr&rrs rJrMzKernel.remove_inplace_buffer;s9 /6*1 !!$'   &rLc t|ttfr|Dcgc]}|j|c}Stj j j|}t|jd}|Dcic]W}t|tjtjtjfr||jj!|Y}}t#||Scc}wcc}w)Nc|jSrYr))ss rJr#z(Kernel.rename_indexing..Hs !&&rL)rs)rlisttuplerename_indexingr1rurrsorted free_symbolsrr UNBACKED_INTSIZEPRECOMPUTED_SIZEr^rr))rr=rsorted_symbols replacementss rJrXzKernel.rename_indexing@s edE] +5:;D((+; ;  ))%0 2 28HI$  %%II)) tyy~~a    %..<  s C%;AC*ct|i|SrY)r )rr^r_s rJrzKernel.create_cse_varWsD+F++rLcZ|y|jj|jS)zC Returns arg name of a given input or output node. N)r^rr)rrs rJrzKernel.arg_nameZs' <yy!!$--/22rL)NT)r^zOptional[KernelArgs]rr^r]r)rr=r]r>r)r$r$r%Optional[IndentedBuffer]r&rbr]r>rrmr=rhr]r rrmr=rhrr r]rrY rrmr=rhrr rIr0r]r rrrqrNrqrOr/r+Union[CSEVariable, tuple[CSEVariable, ...]]r]rgrRrrSzUCallable[[tuple[CSEVariable, ...], tuple[CSEVariable, ...]], tuple[CSEVariable, ...]]rTtuple[CSEVariable, ...]r]ri rRrrTrirXr^rYr^r]ri)r]zdict[sympy.Symbol, sympy.Expr]rTr r]rr^r r_rqr`r^rarrbzOptional[CSEVariable]r]r r) r;zUnion[CSEVariable, str]rBrrCrr9z!Optional[Union[CSEVariable, str]]r]rmr)r=rhr]rmrr rrrmr]r)r=z;Union[list[sympy.Expr], tuple[sympy.Expr, ...], sympy.Expr]r]rh)r^rr_rr]r )rr8r]r)#r_r`rar rrr rnr?r@r"r(rVr+rXrWrTrUrZr4rcrr7r;rDr>rrrErNrMrXrrrrs@rJr r DsM3FC9=I6=PT /( /HL /  /D&&(,'+ F F %F % F  FF8""SW"" *"3>"FO" " """& " ; " 5 ""'" " ( " !""'"(" "  " ! ""4804 " "C "& " $ "  "1 ". "  """37 V $VV V 0 V V<""&0"9="FJ" " "4%)N'' /P/ /.,3rLr c8eZdZUdZded<dZded<dZded <y) rrrz ClassVar[str]rsNrrrrGrmops_name)r_r`rarsrrrrnrdrLrJrrrrcs!"C"#'E 'HcrLrrcb ddl}|j|jS#t$rYywxYw)Nr) undefined)jinja2 EnvironmentStrictUndefined ImportError)rqs rJ jinja2_envruks?!!,,"   s " ..ceZdZdZe d d dZed dZe d dZd dZ ddZ ddZ y)KernelTemplatezg Base class for defining kernel templates. Children classes: TritonTemplate, CUDATemplate c|jd}t|dkDr|ddDcgc] }d|z|z|zc}|dddj|Scc}w)NTr>rrG) splitlinesr|r})source num_indentsindents_spacinglinesr.s rJindent_except_firstz"KernelTemplate.indent_except_first~sd!!$' u:>INqrAE&4<E!"Iwwu~sAct}|ytj|jd<ddlm} |j |S#|$r}Gdd|}|||d}~wwxYw)Nr~r)TemplateSyntaxErrorc(eZdZdfd ZddZxZS)IKernelTemplate._template_from_string..DetailedTemplateSyntaxErrorct||j|j|j|j ||_yrY)rrnmessagelinenorfilenameoriginal_error)rrrs rJrnzRKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__init__s>G$&..&--&++&//  +9D'rLcFd|jd}|d|jdz }t|jdr|jjj d}|dz }t d|jdz }tt||jdz}t||D]s}||jdz k(rN||dzd ||dz }t|jd s=|d d |jjdz zzd zz }c||dzd||dz }u|S)NzError in template at line  zError message: rzz Context: rrr>z: --> columnz rz^ z: ) rrrrrzsplitmaxminr|r9r)r error_infor}startendrs rJrNzQKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__str__sA#=dkk]"!MJODLL>"DDJt22H= $ 3 3 : : @ @ F"l2 #At{{Q 7!#e*dkkAo>!&uc!2 KA DKK!O3 *QveAhZr.J J #*4+>+>#I$.(/*-1D1D1K1Ka1O*P)Q*/)0%&J !+QveAhZr.J J  K&%rL)rrr]rr)r_r`rarnrNrrs@rJDetailedTemplateSyntaxErrorrs 9&rLr)rurwr~filtersrqr from_string)rzenvrers rJ_template_from_stringz$KernelTemplate._template_from_stringsjl ;-;-O-O )*.# 8??6* *"! 8 &.A &>.a0a 7C! 8sAA!AA!ctjjt|tt fr.|Dcic]!}|j |j#c}n |j |jidfd }|Scc}w)Nc>j|}||S|SrY)rR)rrT_get_dtype_reallookups rJrz1KernelTemplate._fake_get_dtype..get_dtypes'ZZ%F! "4( (rL)rrmr]rq)r1rurrrVrWr) fake_outsrnrrrs @@rJ_fake_get_dtypezKernelTemplate._fake_get_dtypesr''++ i$ /AJK#cllncmmo5KF((*I,?,?,ABF ) Ls&B c||_yrYr)rs rJrnzKernelTemplate.__init__s  rLc  |j|jdi|y#t$rQ}tj d|t |tj tjk|cYd}~Sd}~wwxYw)a% Maybe generates a new ChoiceCaller and appends it into existing choices. Returns None if success, otherwise returns the error. choices: A list of ChoiceCallers. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. Nz3Cannot Append Choice: %s. KernelTemplate type is %s) stack_inford) rkrrSrinforgetEffectiveLevelrFINFO)rchoicesr_rs rJmaybe_append_choicez"KernelTemplate.maybe_append_choicesn  NN=4==262 3"  HHET 002W\\A   H s!$ A>AA93A>9A>c t)zM Generates a ChoiceCaller instance from the given arguments. r)rr_s rJrzKernelTemplate.generates "!rLN))rzrmr{rr|rr]rm)rzrmr]r)rzUnion[list[Buffer], Buffer]r]zCallable[[str], torch.dtype]rl)rz list[Any]r_rr]zOptional[NotImplementedError])r_rr]r6) r_r`rarrcr~rrrnrrrdrLrJrwrwws >?"%8; *8*8X. %" ,/ &."rLrwc"eZdZdZdfd ZddZddZ d ddZ ddZddZ ddZ d ddZ dd Z dd Z dd Z dd Z d dd ZxZS)rBcbt|ddlm}||_||_||_y)NrValueRangeAnalysis)rrnrr vr_analysisr'parent_handler)rr'rrrs rJrnzCSEProxy.__init__s+ /-/ ,rLc|jgit|jitddfd }t j |S)Nrctjj[tjjj}|dk(rt j dk(n|dk7rt jdk(nd}nd}|r% dk(r j}nt i }nd}tjjjtjj||}t jjrY|rWddlm}t%|t&t(fr| }tjjj+d |d ||d  d z |j-  |S) Nrrr)Fmasked)rrrr)r*rrrQr>)r1rur~rtrrr!r&rrrwr'rrrrrtorch._inductor.codegen.tritonr*rrVrWr:r) rm device_strtriton_backendrcsevarr*r^r dtype_handlerr_r output_idxrs rJdo_csez!CSEProxy._default..do_csesiww%%1WW@@BGG "U*&&(2"U* ,,8 "'8##(;;L$7%$$' &$'L $ XX\\**  " +F"">>>FlT5M:#/ #;L  **'xz+l:S9TTUV !OJ  ! !$f 5MrL)rmrmr]r )_bound_variablerwrrpytreetree_map) rrr^r_rrrrrs ``` @@@@rJ_defaultzCSEProxy._defaultsh%%%d.D sV11&Vs) set_indirectrTrUct|tr |jSt|tjr t |S|SrY)rr rrrrrs rJ arg_to_boundz.CSEProxy._bound_variable..arg_to_boundM s2a-88O5::.&q>)HrL)rrr]r)rrselect_algorithmrrr1r'rr interpreterrrrdictrRrcompute_all_boundsrr%rVr^rwr) rrr^r_rrr arg_boundsrs @rJrzCSEProxy._bound_variable2 s 0; ahh 4 5&&( (--,, >>T !dkk&@&@&Ldkk88$? ??;;--11';;N;N;PQ Q  & &73Et+LV0UVV"**,,  : c,56J274++T2J? ?""$$rLct|trtj|}t|tjsJ||j j dkr|rtj|tj|tj}|j jdk\r0tj|d}tj|||}n|}tj }|j tj k7rt|tj"r|j tt$ dz}t|j |z|j|z}|j jdk\r"|j tdt$z} || z}|j&j(j+|j&j,||}|j.j1|||} t3|ro|j j dk\ } t|tj" xs|j j|k } |j&j5| || | | S)NrrN)r)rrrr9rrrBr,rrMrlongrCltrrrNumberrr'rrrrr>r#rD) rr;rr<r=stmr new_bounds neg_boundspos sympy_var assert_lower assert_uppers rJr>zCSEProxy.indirect_indexingY s dC ==&D$ +1T1+ ::  a ggc3>>$ #CD::##q(QB))BS1C%,,.Jzz[0022z$ 7U!ZZ+vgr*BB ($$t+Z-=-=-D ::##q(**{1f'==C!+c!1J++//**4;;+>+>J*WC''99#tUK 5 ! # 0 0A 56L)$ ==   4'BL KK $ $YlL QrLc>|jj||||SrY)r'rDrAs rJrDzCSEProxy.check_bounds s{{''dE5AArLc||jjjvr)tjjj |t |tjr|jj||S|jjj}||vr||S|jj||}|jdk(r|jxjdz c_ |Sr)r'rrr1rrrrTMPr+rrVrr)rrr=routs rJrVz CSEProxy.load s 4;;??55 5 HH & & * *4 0 udhh /;;,,T59 9kkoo11 ; t$ $kktU+ ==A  KK A %  rLcl||jjj|<|jjr{|tj j vr^|jjj|}|jD]%}||jjj|<'yyyrY) r'rrrr1runame_to_buffer get_output get_mutations)rrrrn other_names rJ_update_store_cachezCSEProxy._update_store_cache s,1 ##D) ;; # #0F0F(F++**55d;C!//1 @ :? ++J7 @)G #rLc|jjj|||j|||tj j vr|jj||||Sy)N)rI)r'rrrr1rur&rWrHs rJrWzCSEProxy.store sd &&**40 <  $ $T5 1 qww.. .;;$$T5%d$C CrLc|jjj||j|||tj j vr|jj|||SyrY)r'rrrr1rur&rXrKs rJrXzCSEProxy.store_reduction sZ &&**40   u- qww.. .;;..tUEB B /rLc||jxjdz c_|jj||||Sr)r'rrTrMs rJrTzCSEProxy.reduction s4 !!Q&!{{$$UI~uMMrLc<|jj|||SrY)r'rUrQs rJrUz CSEProxy.scan s{{ F;;rLc>|jj||||SrY)r'rZrWs rJrZz CSEProxy.sort s{{ CCrLc D|jj|||||||S)a [Note: Inductor bucketize op] Inputs: ------- values: the values to be bucketized. boundaries: a tuple containing (a) the name of the boundaries tensor (which must be sorted, unless the sorting tensor is present), (b) the length of the tensor in the last dimension (i.e. the length of one set of boundaries), (c) the number of elements in the underlying storage (i.e. the length of the flattened tensor, ignoring striding), and (d) the stride of the tensor in the last dimension. boundary_indices: indices into a flattened version of the boundaries tensor, of the same size and shape as "values". Each index points to the first element in the set of boundaries to be used for the corresponding value. indexing_dtype: the dtype to use when indexing into the boundaries tensor. This must be int64 or int32. This additionally specifies the dtype of the return value. right: see "Details" below. sorter: an optional tuple containing (a) the name of an optional sorting tensor, used to access unsorted boundaries without reordering the boundaries tensor, and (b) the stride of the tensor in the last dimension. The values in the sorting tensor are used as indices into the *last* dimension of the boundaries tensor, with all other indices matching. The size of the sorting and boundaries tensors must be equivalent. sorter_indices: must be present if the sorting array is present; see "boundary_indices" for the equivalent definition for the boundaries tensor. Output: ------- The buckets each value belongs in, within a given set of boundaries. 0 indicates a position before the first boundary, and len(boundaries_set) represents a position after the last boundary. Details: -------- Given a value and a set of boundaries, calculate the bucket that each value belongs to. This works differently in 1-D and N-D cases. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [0, 4, 4, 8], right=True return = [[ 0, 1, 1, 1], [1, 3, 3, 4]]. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [[0, 4], [4, 8]], right=True return = [[ 0, 1, 1, 1], [0, 1, 1, 2]] Note that in the N-D boundaries case, the shape of "values" and "boundaries" must match in every dimension _except_ the last. When right == False, bucket i refers to range (boundaries[i], boundaries[i+1]]. When right == True, bucket i refers to range [boundaries[i], boundaries[i+1]). Boundaries must be non-decreasing, or a sorter must be provided which would re-index offsets in a non-decreasing order (e.g. the second output of torch.sort(offsets)). Otherwise, the result is undefined. )r'rcr\s rJrczCSEProxy.bucketize s1L{{$$         rL)r'z Kernel[Any]rzOpsHandler[Any])rrmr^ztuple[Any, ...]r_zdict[str, Any]r]r)rrmr^rr_rr]rr) r;r rrr<r^r=r^r]rrrc)rrmrr r]rrYrerdrfrhrjrrk)r_r`rarrnrrr>rDrVrrWrXrTrUrZrcrrs@rJrBrBs D-;.z%%V / /%/ /  /  /bBB&0B9=BFJB B "@SW *3>FO CNNN& N ; N 5 N <' <  <( < ! <D'D(D D  D ! D 4804N N CN & N $ N  N 1N .N  N rLrB)rIrmr]rrY) rlrmrrrrrrr]r)rlUnion[torch.device, str, None]r]zOrderedSet[BackendFeature])rlrr rr]r^)rlrmr]zOptional[SchedulingConstructor])F)rlrmrr^r]rr)r=Sequence[sympy.Expr]r>rr?rr]r)rlrmrBrr]r)rlrmr]r)r]rmr^rr_rr]r)rrmr]r^rdr)r]r) __future__rr? dataclassesenumr~rrFrrretypingrrrrrr r r r r rrtyping_extensionsrrrtorch.fxtorch._prims_commonr torch.utilsrrtorch.utils._ordered_setrtorch.utils._sympy.numbersrtorch.utils._sympy.printersr_PythonPrintertorch.utils._sympy.symbolrrrtorch.utils._sympy.value_rangesrrrGrrdtype_propagationr ops_handlerrr utilsr!r"r#r$r%r&r'r(r)r*r+ virtualizedr,r-r.r/r0r1collections.abcr2r3r4rr5r6r7r8rr:rr;r<r=r2r@rArrrrmrrr_logginggetArtifactLoggerr_rD getLoggerrrKrNrg dataclassrrrrr KernelArgTyperrrrrrrr rr lru_cacherr@rCrIbfloat16r[float16r^rfloat64int8int16rr\ruint16r|uint64rJrarcrcompile IGNORECASEr rrrr INT_TO_FLOATr}r*r,r4rCrFrKrMrPr rrrWrrReductionCacheKeyrrr rrrurwrBrs0rJr s8 "      & ?)/-GOOD:;    QPBB>>$DD- B$hy&9%:N%JK23sELL()JF~~//*E g!= / /(T[[[| ###    === lIw8H,VW ,.).3"3"l:<6;8@D  ,/!=    &T &3 *33$3 *35C3 3U &+   " ! TJJZU U$U U U; ;&7; ; , NNEKK MM5;;> JJ MM MM JJ KK KK KK KK LL LL LL    u >:,' ' '' 'T_B_BD%N%S1S1l"rzz";2==Q a;#%5z#a;H  - - - 6:_6 ;HH -  _6 ;HH /-  _6;HH /- _6$;HH /- %_60;HH /- 1_6< ;HH *)   =_6L ;HH (0   M_6X ;HH 66>   Y_6h ;HH 1  i_6r ;HH 2  s_6|;HH 1 }_6F;HH 2 G_6P;HH %8$  Q_6^ ;HH &%   __6j;HH %8  k_6v ;HH &  w_6@;HH + A_6L%;HH 88 ) M_6X%;HH 88 ) Y_6d%;HH 8 )e_6n%;HH 8 )o_6z ;HH '  {_6D ;HH (  E_6N;HH S O_6\,;HH ? 0]_6f,;HH ? 0g_6r&;HH 9 *s_6| ;HH *  }_6F);HH D -G_6P);HH D -Q_6Z);HH D -[_6d);HH D -e_6n(;HH C ,o_6x$1;HH L 5$y_6B$1;HH L 5$C_6L$1;HH L 5$M_6V$1;HH L 5$W_6`';HH B +a_6j(;HH C ,k_6t(;HH C ,u_62_D -#-">*Z  NNN ,AAH !;!;H5 +; T  k5c!1223 5a'/=0 1aH < <\3Wgo.\3~ Tp"p"fz ~z w7s c