
    Vh                   o   U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZmZ d dlmZmZmZmZmZmZmZ d dlmZ d dlmZ d dlZd dlZd dlZd dl Zd dl!m"c m#Z$ d dl%m&Z& d d	l'm(Z( d d
l)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6 d dl7m8Z8m9Z9 d dl:m;Z; d dl<m=Z=m>Z>m?Z?m@Z@mAZA ddlBmCZC ddlDmEZEmFZFmGZGmHZH ddlImJZJmKZK ddlGmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZY ddl"mZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZd ddlemfZfmgZg erddlhmiZi  ed      Zj ed      Zk e;ddg      Zl ej                  en      Zoi Zpdeqd<   i Zrdeqd<    e;ej                  j                            Zuej                  j                  Zvej                  j                  Zwej                  j                  Zx e;ej                  j                            Zy e;ej                  j                     ej                  j                  g      Z| e;ej                  j                            Z}i Z~d eqd!<   ej                  j                  Zd" ZdVd#ZdWd$Zd% Zd& Zd' Zd( Z eevj                  evj                  evj                  evj                  evj                  evj                  evj                  evj                  evj                  evj                   evj"                  evj$                  evj&                  g       ej(                  ej*                  ej,                  ej.                  ej0                  ej2                  ej4                  ej6                  ej8                  ej:                  ej8                  ej<                  ej>                  d)ZdXd*Zd+ Zd, ZdYd-Zd. Zd/ Z	 	 	 	 	 	 	 	 	 	 	 	 dZd0Zd1 Z	 	 d[d2Zd3e1jT                  d3f	 	 	 d\d4Zd5 Zd]d6Z	 	 	 	 	 	 d^d7Zd_d8Zd_d`d9Z eej                  j                  d:      d;        Z{ eexjb                  d:      d`d<       Zd3d=d`d>Z eevjh                  jj                  d:      d`d?       Zd3d3d@dadAZ eexjp                  d:      d_dadB       ZddCe1jT                  d3ddd3d3df	dDZdE Z e        	 d_dFZ eevjz                  d3dG      dH        Z eevj|                  d3dG      dI        Z eevj~                  evj                  evj                  evj                  exj                  g      dJ        Z eevdK      r  eevj                        eī        eevj                  d:      dbdL       Z eevj                  d:      dbdM       Z eevj                  g      dbdN       Z eevj                        dO        Z eevj                        dP        Z eevj                        dQ        Z eevj                        dR        Z eevj                  j                        dS        Z eevj                        dT        Z eevj                  d:      dU        Z eexj                  d:      dV        Z eevj                  d:      dW        Z eevj                        dX        Z eevj                  d:       eevjh                  d:       eevj                  d:      dY                      Z eevj                  d:      dZ        Z eevj                  d:      dcd[       Z eevj                  d:      dbd\       Z eevj                  d:      dbd]       Z eevj                  d:      dbd^       Zddd_Z eej                  d:      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ded`       Z eej                  d:      dda	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dfdb       Z eej                  j                  d:      	 	 	 	 	 	 	 	 	 	 	 	 	 	 dgdc       Z eej                  j                  d:      dda	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dhdd       Z eej                  j                  d:      	 	 	 	 	 	 	 	 	 	 	 	 	 	 dide       Z eej                  j                  d:      dda	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 djdf       Z eevj                        dddg       Z eevj                  d:      dkdldh       Z eevj                  d:      dkdldi       Z eevj                  d:      dkdldj       Z eevj                  d:      dk        Z eevj                  d:      dddl       Z eevj                  d:      dddm       Z eevj                  d:      dddn       Z eevj                  d:      do        Z eevj                  d:      dp        Z eevj                  d:      dq        ZdddrZ eevj                        dmdt       ZdnduZ ej                  d      dv        Zd]dodwZd]dodxZdndpdyZdqdzZd{ Z eej                  j                  j                  d:      d|        Z eevj                  d:      d}        Z eevj                  d:      d~        Z eevj                   j                  d:      d        Z eevj                        d        Z ej                  d      d        Zd Z eevj                  j                        Z eevj                  j                        Z	 eevj                  j                        Z eevj                  j                        Z eevj                          eevj                        d        Z eevj                        d        Z
 eeFj                  d:      d        Z eeFj                   d:      drd       Z eeFj$                  d:      d        Z eeFj(                  d:      d        Z eeFj,                  d:      d ddsd       Z eeFj                  d:      d d	 	 	 	 	 	 	 	 	 dtd       ZdudZdvdZ eevj6                  j8                  d:      d3d3ddd	 	 	 	 	 	 	 	 	 	 	 	 	 dwd       Z eevj:                  d:      d3d3d	 	 	 	 	 	 	 dxd       Zd Zd Zd Z d Z!d Z"d Z# eevjH                          eevjJ                          eevjL                          eevjN                          eevjP                  d3        eevjR                  j                  d3        eevjT                          eevjV                  d3        eevjX                          eevjZ                          eevj\                          eevj^                  j                          eevj`                  j                          eevjb                          eevjd                  jf                          eevjh                  j                          eevjj                  j                          eevjl                          eevjn                  d3        eevj                  e"        eevjp                  e        eevjr                  e        eevjt                  e        eevjv                  e        eevjx                          eevjz                          eevjz                          eevj|                          eevj~                          eevj                          eevj                  e        eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                  e        eevj                          eevj                  e        eevj                          eevj                          eevj                  j                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                  j                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                          eevj                  j                          eevj                  j                  d3        eevj                  e        eej                  j                  j                          eej                  j                  j                          eej                  j                  j                          eevj                          eevj                          eevj                  e        eevj                          eevj                          eevj                          eevj                           eevj                          eevj                  j                          eevj                  j                  e#d3        eevj
                  j                  e#d3        eevj                  j                  e#d3        eevj                  j                  e#d3        eevj                  j                  e#d3        eevj                  j                  e#d3        eevj                  j                  e#d3        eevj                  j                  e#d3        eevj                  j                  e#d3        eevj                  j                  e#d3        eevj                  j                  e#        eevj                  j                  e#        eevj                   j                  e#        eevj"                  j                  e#        eevj$                          eevj&                  d:      d_d       Z eevj(                        ddd       Zd Z eevd      r  eevj,                        e        eexj.                        d        Z eevj0                  d:      dyd       Z eevj2                  d:      dzd       Zd Z eej                  evj6                  g      dddd3dd       Z eej8                        d]d       Z eej:                        d        Z eevj>                        d        Z eevj@                        d        Z eevjB                        	 d{dddd       Zd Z eevjF                  d:      d        Zd Z eejJ                  evjJ                  g      dddddddd       Zd Zd Z  eevjP                         ee            Z e ed            Z e ed             Zd Z eevjX                        dddddd       Z eevjZ                        dddddd       Z eevj\                        dddddd       Z eexj^                  j                        d        Z eej`                  evj`                  g      d        Z eevjb                  d:      d_d       Z eevjd                  d:      d|d       Zd Z	 dndZd ZdndZ eevjn                  d:      d        Z eevjp                  d:      d        Z eevjr                        d_d       Z eevjt                        d_d       Zd ZdÄ Z eevjz                  d:      d_dĄ       Z eeFj|                  d:      d_dń       Zd_dƄZ eevj                  j                  d3ǫ      Z eevj                  j                  d3ǫ      Z eevj                  d:      dȄ        Z eevj                  d:      dɄ        Zedʄ        Z eevj                  d:      dbd˄       Z eevj                  d:      d}d̄       ZddCd͜	 	 	 	 	 	 	 d~d΄Z eevj                  d:      ddϜddЄ       Z eevj                  d:      d}dф       Z eevj                  d:      d}d҄       Z eevj                  d:      d}dӄ       Z eevj                  d:      dCdԜddՄ       Z	 	 d	 	 	 	 	 ddքZ eevj                  j                        dbddׄ       Z eevj                  j                        dbdd؄       Z eevj"                  j                        	 d]	 	 	 ddل       Z eevj$                  j                        	 d]	 	 	 ddڄ       Z eevj                  j                        	 	 	 d{	 	 	 	 	 ddۄ       Z eevj                  j                        	 	 	 d{	 	 	 	 	 dd܄       Zd݄ Z eexj                  j                        dބ        Z	 	 	 	 	 	 	 	 dd߄Z eevj                  d:      ddd       ZՐddZ֐ddZאddZ	 ddZd Zd ZdddZ	 d_dZ eexj                  d:      	 d_d       Z eexj                  d:      d        Z eevj                  j                  d3ǫ      Z eevj                  d:      	 	 	 	 dd       Z eevj                  j                  d3ǫ      Z eevj                  d:      d        ZddZd Zd Zd Z eevj                  j                  d3ǫ      Z eevj                        d        Z eevj                  j                  d3ǫ      Z eevj                        d        Z eevj                  j                  d3ǫ      Zd Z eevj                        d        Z eevj                  j                        	 dd       Z eevj                  j                  d3ǫ      Z eevj                  j                  d3ǫ      Z eevj                  d:      	 	 	 	 	 dd       Z eevj                  d:      	 	 	 	 	 dd       Zd Z eevj                  j                  d3ǫ      Z eevj                  d:      	 dbd       Z eevj                  j                  d3ǫ      Z eevj                  d:      	 dbd       Zd Zd ZdbddZd Z eevj                        ddd d       Zd Zd Zd Zd Z eevj                  exj                  g      dbdd3dd       Z eevj                         dbdd3dd       Z d	 Zed
        Z eevj                  j                  d3ǫ      Z eevj                  j                  d3ǫ      Z eevj                  j                  d3ǫ      Z	 eevj                  dC      d        Zd_dZ
 eevj                        d        Z eevj                  d:      d_d       Zed        Zed        Z eevj                  dC      dbd       Z eevj"                  gdC      d        ZddZ eexj                  gdC      d        Z eevj(                  evj                  j8                  gdCe1j*                  G      d        Z eevj,                  exj,                  gdC      d        Z eevj.                  exj.                  g      ddd d       Z eevj2                  j                        Z eevj6                  j                        Z eevj:                  j                        Z eevj>                  j                        Z  eevjB                  j                        Z" eevj2                        d]d       Z eevj6                        d]d       Z eevj:                        d        Z eevj>                  d:      dbd       Z eevjB                  d:      dbd       Z! eevjF                        ddd d       Z# eevjH                        dd       Z% eevjL                  d:      dd        Z' eevjP                  d:      dd!       Z)  eexjT                         ed"               eevjV                         ed#            Z,  eevjZ                         ed$            Z.  eevj^                         ed%ej0                  &            Z0  eevjb                         ed'ej0                  &            Z2 eevjf                  dCd()      Z3 eevj                  j                  d3ǫ      Z4 eevj                  j                  d:      ddsd3d*d+       Z5 eevj                  j                  d:      dd,       ZNd]d-Z6d. Z7 e6evjp                        Z8 e7evjr                        Z9 e6evjt                        Z: e6evjv                        Z; eevjx                        Z< e7evjz                        Z= e7evj|                        Z> eevj~                        Z? eevj                  dC/      Z@ e7evj                          e7evj                          eevj                        ZC eevj                        ZD eevj                        ZE eevj                  d01      ZF eevj                        ZG eevj                        ZH eevj                        ZI e6evj                          e6evj                        ZK  eevj                  e1j*                  :      eK        e6evj                          e6evj                          e6evj                          e7evj                          eevj                  ddCej<                  2      ZP eevj                  ddCej<                  2      ZQ eevj                  ddCej<                  2      ZR eevj                  ddCej<                  2      ZS eevj                        ZT eevj                        ZU  eevj                        eT         eevj                        eU        eevj                        ZX eevj                        ZC e6evj                        ZY eevj                          eevj                  d31      Z[ eevj                          eevj                  ej<                  &         eevj                        eX        eevj                  ej<                  &        eevj                  ej<                  &        eevj                  ej<                  &        eevj                  ej<                  &      Za eevj                  ej<                  &        eevj                  ej<                  &        e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                          e6evj                         dd4ltmuZumvZv d5 ZwevD ]S  Zx ewevex      D ]  \  ZyZzZ{ eeyexeze{6         ewexex      D ]  \  ZyZzZ{ eeyexeze{6        U  eevj                  j                  e3dC/      Z~ eevj                  j                  e3dC/      Z eevj                  j8                  e3dC/        eevj                   j                  e      Z eevj                   j8                  e        eevj                   j                  e      Z eevj                  j                  e@        eevj                  j                  e@        eevj                  j                  eX        eevj
                  j                  eC        eevj                  j                  e        eevj                  j                  e        eevj                  j                  e        eevj                  j                  e      Z eevj                  j8                  e        eevj                  j                  e      Z eevj                  e>        eevj                  e8        eevj                  j                  eT        eevj                  j                  eT        eevj                  j                  eU        eevj                  j                  eU        eevj                  j                  eT        eevj                  j                  eT        eevj                   j                  eU        eevj                   j                  eU        eevj"                  eY        eevj$                  e[        eevj&                  e       d7 Z eevj*                  j                  evj                  j                  e~        eevj*                  j                  evj                  j                  e        eevj,                  j                  evj                   j                  e        eevj,                  j                  evj                   j                  e        eevj.                  j                  evj                  j                  e        eevj.                  j                  evj                  j                  e       d8 Z eevj2                  e3        eevj4                  eD        eevj6                  eE        eevj8                  eF        eevj:                  eG        eevj<                  eH        eevj>                  eI        eevj@                  e        eevjB                  j8                  e        eevjB                  jD                  e        eevjF                  eP        eevjH                  eQ        eevjJ                  eR        eevjL                  eS        eevjN                  e@        eevjP                  e<        eevjR                  e=         eevjT                        eD         eevjV                        eE         eevjX                        eG         eevjZ                        eH         eevj\                        eI        eevj^                  evjT                          eevj`                  evjV                          eevjb                  evjX                          eevjd                  evjZ                          eevjf                  evj\                          eevjh                        d]d9       Z eevjj                  jl                        d:        Z eevjn                  jl                        d;        Z eevjp                        d<        Z e8jr                         D ]  \  ZZ  e e9e            e         eejx                        d=        Z eevj                        d>        Z eej                  j|                  j~                        d?        Z eej                  j                  j                        d@        Z eej                  j                  j                  j                        dA        Z eej                  j                  dB      r; eej                  j                  j                  j                        dC        Z eej                  j                  j                        dddD       Zd dElɐmʐZ  eeʫ        ee*      dF        Z eej                  j                  j                  d:      dG        Z eej                  j                  j                  d:      dH        Z eej                  j                  j                  d:      ddI       Z eej                  j                  d:      ddJddK       Z ee(d:      	 	 	 	 ddL       Z eej                  j                  j                  j                        dM        Z eej                  j                  j                  d:      dN        ZddOlՐm֐Z  e֫         eeFj                  d:      dP        ZddQlDmؐZ  eCeث       ddRlDmِZ  eِj                           eِj                          ddSlDmܐZ  eܐj                          ddTlDmސZ  eސj                          ej                  ddU       Zy(      )annotationsN)defaultdict)IterableSequence)AnyCallablecastOptionalTYPE_CHECKINGTypeVarUnion)	ParamSpec)patch)counters)associative_scan_op)triton_kernel_wrapper_mutation)canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDget_computation_dtypeis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)
OrderedSet)CeilDivFloorDivIdentity
IntTrueDivModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)	DtypeView
ExpandViewIndexingConstantIRNode	is_tritonOnlineSoftmaxReductionops_wrapperPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)ceildivdecode_device
is_dynamicis_gpuis_pointwise_useis_view,needs_fallback_due_to_atomic_add_limitationspad_listlike#register_op_dtype_propagation_rulessympy_productuse_scatter_fallback)opsV)ReductionType_T_Pztorchvision::roi_alignzaten::index_addz8dict[Union[Callable[..., Any], str], Callable[..., Any]]	loweringsz9dict[torch._ops.OpOverload, Optional[Callable[..., Any]]]_maybe_layout_constraintsz2dict[torch._ops.OpOverload, torch._ops.OpOverload]inplaceable_foreach_opsc                     t         j                  j                  j                  D ]7  } | j                  D ]&  }|j                  dk(  r|j
                  t        v r%  y 9 y)Ncall_functionTF)rI   graphcurrent_nodeusersoptargetforeach_ops)nodeusers     H/home/dcms/DCMS/lib/python3.12/site-packages/torch/_inductor/lowering.pycur_node_has_non_foreach_usersr[   |   sS    $$** JJ 	DGG.DKK;4N	
     c                f   t        t              }d}t        |       D ]  \  }}t        |t              sd}|f}t        |  xs t        j                  }d }|D ].  }t        |t              s|j                  j                         } n |J d       |r|\  }|||f   j                  ||f        |S )NFTz.foreach op should have at least one tensor arg)r   list	enumerate
isinstancer   r?   r)   #combo_kernel_foreach_dynamic_shapesr:   data
get_deviceappend)	arg_pairsoutunpack_argsiargsuse_foreachdevicets           rZ   group_foreach_argsrm      s    
d
CKY' 54$)K7DD!!OV%O%O 	  	A!Y'**,	 !S#SS!GTV[!"))1d)45  Jr\   c                     t         t        j                  j                        sy t        v r	t            S  t
        v r
dt         <   y fd}t               } ||      S )zHGet layout constraints. Returns None if there are no layout constraints.Nc                    | t         j                  j                  j                  u rt        t
        <   t
           S | t         j                  j                  j                  u r
d t
        <   y t        d|        )NzUnknown layout constraint tag: )torch_CTagneeds_fixed_stride_orderconstrain_to_fx_stridesrN   flexible_layoutAssertionError)tagfns    rZ   handle_layout_constraint_tagz>maybe_layout_constraints.<locals>.handle_layout_constraint_tag   sf    %((,,777,C%b),R00EHHLL000,0%b) #B3%!HIIr\   )r`   rp   _ops
OpOverloadrN   rM   get_layout_constraint_tag)rx   ry   rw   s   `  rZ   maybe_layout_constraintsr}      s`    b%**//0	&&(,,	Y(,!"%J $B
'C',,r\   c                   t         j                  j                  j                  t         j                  j                  j                  g}|D ]  }|| j
                  v s|c S  t         j                  j                  j                  |       r$t         j                  j                  j                  S t        t         j                  j                  t        j                        S N)rp   rq   rr   rs   ru   tags_libraryutils
is_builtingetattrr)   #custom_op_default_layout_constraint)rx   tags_by_priorityrw   s      rZ   r|   r|      s    --$$   "''>J ~~&&r*xx||+++588<<!K!KLLr\   c                $    | st        d|       y )Nzinductor does not support NotImplementedError)condmsgs     rZ   
assert_nyir      s    !$>se"DEE r\   c                N    t         t        t        t        t        f      r D cg c]  }t        |       c}S t        j                          t         t        j                  j                        r-t        j                   fd j                         D               y y c c}w )Nc              3  6   K   | ]  }t        |        y wr   )r   ).0overloadrx   s     rZ   	<genexpr>z,add_needs_realized_inputs.<locals>.<genexpr>   s      %
&.GB!%
   )r`   r^   settupler    add_needs_realized_inputsneeds_realized_inputsaddrp   rz   OpOverloadPacketupdate	overloads)rx   xs   ` rZ   r   r      sz    "tS%45689)!,99b!"ejj112$$ %
24,,.%
 	
 3 :s   B"c                    t        | t        j                  j                        r)| j	                         D ]  }|t
        t        | |      <    y |t
        | <   y r   )r`   rp   rz   r   r   rN   r   )rx   
constraintr   s      rZ   add_layout_constraintr      sL    "ejj112 	JH?I%gb(&;<	J )3!"%r\   )r   r(   r&                     	   
         c                b    t        | t              s| S | t        v sJ d|  d       t        |    } | S )Nzid z missing from DTYPE_ID_LOOKUP)r`   intDTYPE_ID_LOOKUPdtypes    rZ   decode_dtyper     s=    eS!O#Os5'1N%OO#E"ELr\   c                    t        | t              r4t        | j                               xs t	        | j                               S t        | t
        j                        r| j                  du S t        | t              S )NT)	r`   r:   r   	get_dtyper   sympyExpr
is_integerr   r   s    rZ   is_integer_typer     sX    !Y.Q2B1;;=2QQ	Auzz	"||t##!S!!r\   c                t    t        | t              rt        | j                               S t        | t              S r   )r`   r:   r   r   boolr   s    rZ   is_boolean_typer     s*    !Y..!T""r\   c                \    d }|D cg c]
  } ||       }}t        |d| i\  }}|S c c}w )Nc                    t        | t        t        j                  f      r| S t	        | j                               }t        j                  dg|z  | j                               S )Nr(   r   )	r`   r   r   Basiclenget_sizerp   zerosr   )inpdims     rZ   construct_inputz+get_promoted_dtype.<locals>.construct_input!  sG    cFEKK01Jclln%C;;sSy@@r\   type_promotion_kind)r   )r   ri   r   arginps_r   s          rZ   get_promoted_dtyper      s@    A -11SOC 1D1!4Q=PQHAuL 2s   )c                0   t        | t        t        f      s| g} nt        |       } t        |       D ]b  }t        |t        j                  j
                        s(|j                         D ](  }t        ||      }|t        vs| j                  |       * d | S r   )
r`   r^   r   rp   rz   r   r   r   rM   rd   )aten_fnrx   r   other_fns       rZ   get_overloadsr   .  s    ge}-)w-7m -b%**556LLN -"2x09,NN8,-- Nr\   c                    t        | t        j                  j                        r|| j                  v S t        | t        j                  j
                        r|| j                         v S yNF)r`   rp   rz   r   _qualified_op_namer{   name)rU   	namespaces     rZ   in_namespacer   >  sO    "ejj112B1111	B

--	.BGGI%%r\   c           	     V    t               D cg c]  \  }}t        |t              s| }}}j                         D 	cg c]  \  }}	t        |	t              s| }
}}	|s|
s fS |s|r|rt        j
                  nl D cg c]0  }t        |t        t        j                  f      st        |d      r|2 }}|j                  d j                         D               t        |d|i|r |d      n|
d      j                         fd} D cg c]
  } ||       c} j                         D 	ci c]  \  }}	| ||	       c}	}|r1t        t        t!        j"                   fd|D        fd|
D                     }t        |d   j%                               }t'        ||d t)        |             D ]
  \  }}| |<    t'        |
|t)        |      d        D ]
  \  }}||<    t+        t)                     D ]<  }t         |   t,        j.                        s!t1        j2                   |   |       |<   > D ]<  }t        |   t,        j.                        s!t1        j2                  |   |      |<   >  fS c c}}w c c}	}w c c}w c c}w c c}	}w )Nr   c              3  :   K   | ]  }t        |d       s|  yw)r   N)hasattr)r   as     rZ   r   z!transform_args.<locals>.<genexpr>^  s     !T7@S!!Ts   r   r   c                    t        | t              rt        |       S t        | t        j                        r"t        j                  | j
                        S | S )Nvaluer   rk   )r`   r:   to_dtyper+   Constantr   )r   rk   r   s    rZ   promoteztransform_args.<locals>.promotei  sD    #y)U++C-{{%OO
r\   c              3  (   K   | ]	  }|     y wr    )r   rh   ri   s     rZ   r   z!transform_args.<locals>.<genexpr>x  s     3T!W3   c              3  (   K   | ]	  }|     y wr   r   )r   kkwargss     rZ   r   z!transform_args.<locals>.<genexpr>y  s     71VAY7r   )r_   r`   r:   itemsrp   r   r   r   r   r   extendvaluesr   rc   broadcast_tensorsr^   	itertoolschainr   zipr   ranger+   r   r0   create)ri   r   	broadcastr   convert_input_to_boolrh   r   args_indicesr   vkwargs_indicesr   promoting_argsr   broadcastedsizerk   r   s   ``              @@rZ   transform_argsr   F  s    #,D/N$!QZ95MANLN$*LLNODAqjI6NaONOV|3 JJE
 a&%++!6771g;N N  !!!TV]]_!TT&$7E &2Da!vnQ>O7P
*, 	
	 %))q
),2LLN;DAq!WQZ-;'3l377
 KN++-.k2EC4E&FG 	DAqDG	C4E4G(HI 	DAqF1I	 s4y! 	;A$q'2;;/$++DGT:Q	;  	?A&)R[[1&--fQi>q		? <} OO2 *;s(   JJJJ	5JJ  J%c                    t        j                        fd       }t        |       }t        j	                  |       t
        j	                  t        j                  ||             |S )a  
    Add a foreach lowering to lowerings dict.

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                 N    t        |       dk  sJ  | i |}t        |       |S )Nr&   )r   r;   )ri   r   rf   	decomp_fns      rZ   wrappedz+_register_foreach_lowering.<locals>.wrapped  s/    4yA~~((C
r\   )	functoolswrapsr   rW   r   rM   dictfromkeys)r   r   r   aten_fnss    `  rZ   _register_foreach_loweringr     sX     __Y   W%Hx T]]8W56Nr\   c                     t        j                         fd       }t                t        j	                  t
        j                   |             |S )a  
    Add a lowering to lowerings dict

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                 l   t        |       } t        |      }d}t        |       dk(  r)t        | d   t         t        f      rd}t        | d         } t        d D              s't        d |j                         D              rJ d       t        | |      \  } }|r| g}  | i |}t        |       |S )NFr(   r   Tc              3  H   K   | ]  }|t         v xs t        |d         yw)_c10d_functionalN)	fallbacksr   )r   rx   s     rZ   r   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s)      
JLR9_DR1C DD
s    "c              3  &   K   | ]	  }|d k(    yw)rf   Nr   r   r   s     rZ   r   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s     =!1:=   zout= ops aren't yet supported)
r^   r   r   r`   r   allanykeysr   r;   )	ri   r   unpackedrf   r   r   r   r   r   s	       rZ   r   z#_register_lowering.<locals>.wrapped  s    t*!%ft9>ja4-@HQ=D 
PW
 
 =v{{}== /= &&)%8:O
f 6D((C
r\   )r   r   r   rM   r   r   r   )r   r   r   r   r   r   s   ````` rZ   _register_loweringr	    sJ    $ __Y  : G$GT]]7G45Nr\   Fc                >    t        j                  t        | |||      S )z+
    Shim to support decorator syntax.
    r   r   r   )r   partialr	  )r   r   r   r   s       rZ   register_loweringr    s&     /3 r\   c                p   g }t        j                  t        |       t        |      t        j                  j
                        D ][  \  }}t        j                  j                  j                  j                  t        j                  |d      d      r|j                  |       bt        j                  j                  j                  j                  t        j                  |d      d      r|j                  |       t        j                  j                  j                  ||       t        t        j                  |      j                         t        t        j                  |      j                         k  r|j                  |       K|j                  |       ^ t#        t        |            S )z
    Broadcasting logic based on symbolic shapes.

    We give the shapes 0 and 1 concrete values, while all other shapes
    are symbolic sympy formulas.
    )	fillvaluer(   Tsize_oblivious)r   zip_longestreversedr   SOnerI   rR   sizevars	shape_envevaluate_exprEqrd   guard_equalsr   expandfree_symbolsr   )r   boutputr   ys        rZ   broadcast_symbolic_shapesr     s-    F%%hqk8A;%''++V !177%%33HHQN4 4 
 MM!WW''55HHQN4 6 
 MM!GG))!Q/5<<?//03u||A7S7S3TTa a ! &!""r\   c                V   |	|J d       ||t         j                  }t        d | D              s| S t        d | D              r*|xs t	        | d|ifd}| D cg c]
  } ||       c}S t        d | D              }g }| D ]  }t        |t        t        f      ro|j                  t        j                  t        j                  ||j                         |j                               t!        |j#                                            t        |t$        j&                        rf|j                  t        j                  t)        ||j                         |j                               t!        |j#                                            	|j                  |        |S c c}w )	NzEonly one of override_return_dtype or type_promotion_kind may be givenc              3  f   K   | ])  }t        |t        j                  t        t        f       + y wr   )r`   r   r   r   floatr  s     rZ   r   z$promote_constants.<locals>.<genexpr>  s"     HAz!ekk367H   /1c              3  f   K   | ])  }t        |t        t        t        j                  f       + y wr   )r`   r   r#  r   r   r  s     rZ   r   z$promote_constants.<locals>.<genexpr>  s"     
D:a#uekk23
Dr$  r   c                    t        | t        j                        r!t        j                  | t        d             S t        j                  | t        d             S )Nindexr   rk   r   )r`   r   r   r+   r1   r>   r   r   r   s    rZ   
const_funcz%promote_constants.<locals>.const_func  sJ    !U[[)**5t1D  {{%d@STTr\   c              3  l   K   | ],  }t        |t        t        t        j                  f      s)| . y wr   )r`   r:   r0   r+   r   r  s     rZ   r   z$promote_constants.<locals>.<genexpr>#  s$     WA:a)Z1U#VaWs   *44r   r'  )r   DEFAULTr  r  r   nextr`   r   r#  rd   r0   r   r+   r   r   get_device_or_errorr^   r   r   r   r1   )inputsoverride_return_dtyper   r*  r   exrf   r   s          @rZ   promote_constantsr2    s    (,?,G OG $)<)D=EEHHH

DV
DD% 
);*
)<*
	U (..!
1..	WW	WB
C a#u&JJ!!KKr||~b>T>T>V '	 5;;'JJ!!$r||~b>T>T>V '	 JJqM), J3 /s   F&c           	     2     d dd fd}|S )Nalphac           
       	
 t        d D              r	rJ   S t              r%| '| dk7  r"t              t        d   |       d<   n| J D cg c]  }|j	                          c}
d   j                         xs d   j                         t        t        d   j                               j                        	dd  D ]`  }t        |t        j                        rt              t        |j                               k(  rDJ d d d|j                                  t        j                   t        j"                  ft$        j&                  d uxr t)        t$        j&                  dd       d uxrb t$        j&                  j*                  j,                  d uxr: t$        j&                  j*                  j,                  j/                  dd	      xr v 	
f
d
}sSd }D ]7  }t        |j                         j                        s'|j                         } n |sd   j                         }xs }t1        j2                  ||      S c c}w )Nc              3  V   K   | ]!  }t        |t              xr t        |       # y wr   r`   r2   r3   )r   r   s     rZ   r   z0make_pointwise.<locals>.inner.<locals>.<genexpr>H  s(      /
;>JsF#6	#6/
   ')r(   r   zndim mismatch  rS   low_precision_pointwise_barrierFc                t  
 t        |       t              k(  sJ d|  d        t        j                  k(  r D cg c]
  } ||        c} S r.r,t        j                  k(  r D cg c]
  } ||        c} S g }t	              D ]e  \  }} ||       }
|   j                         }r2|v r.t        j                  ||d      }t        j                  ||      }|j                  |       g  	| }r.t        j                  |d      }t        j                  |      S |S c c}w c c}w )Nzwrong ndim r;  F)use_compute_types)	r   rp   r   float64r_   r   rH   r   rd   )r(  loadinputs_loaded	inp_indexrf   	inp_dtypedowncastr   emulate_precision_castsrx   r/  is_gpu_deviceloaders	low_pr_fpoverride_fn_when_gpu_float64override_fn_when_input_boolrangess          rZ   inner_fnz/make_pointwise.<locals>.inner.<locals>.inner_fnk  s<   u:V,LE7!F8.LL,

"'B'N2W4UTT%[4UVV,!U]]*3g5Vdd5k5VWW "'0'9 .OItu+C &y 1 ; ; =I.9	3I#&<<YRW#X!ll8Y?!((-. -(*  #||C%PH<<%88
/ 5V 6Ws   D01D5rk   r   rL  rK  )r  r2  r^   mulmake_loaderr   r   r@   r>   rc   typer`   r+   BaseConstantr   rp   bfloat16float16rI   rR   r   rS   metagetr7   r   )r5  r/  r   otherrL  rk   rh   r   rE  rF  rG  rH  rK  allow_alpharx   override_devicerI  rJ  r0  triton_fallbacks    `     @@@@@@rZ   innerzmake_pointwise.<locals>.innerG  sf   &3 /
BH/
 ,
 #"?"F++"6+@A UaZf U3r
= =,23q1==?3##%%>)<)<)>}VAY-A-A-CDIIJABZ 	AEeR__5V I : At1VHAenn.>-?@A 	A ^^U]]3	GG4 #6dB#$$))5# $$))--.OQVW# " 	 	 	8 F !,,.--.\\^F --/ *F	
 	
{ 4s   !I3)r/  r:   r   )rx   r0  rX  rJ  rI  rW  rY  rZ  s   ``````` rZ   make_pointwiser[  >  s     )- P
 P
d Lr\   c                     ddd fd}|S )Nr(   r4  c                   t        t        j                  j                  j                        dk(  xs6 t        j                  j                  j
                  t        v xs
 t               }d }|D ]  }t        |t        t        f      s|} n |J d       g }|D ]H  }t        |t        t        f      s|j                  |gt        |      z         8|j                  |       J t        t        |       }d gt        |      z  }|j                         D ]  \  \  }}	}
g }|
D ]~  \  }}r	 |d| i}n | }|||<   t        j                  j                  |t         j"                        sJ|	sM|sP|j%                          |j                  |j'                                 |st        j                  j)                  |        t+        d |D              sJ |S )Nr   z1at least one input must be a list to a foreach opr5  c              3  $   K   | ]  }|d u 
 y wr   r   r  s     rZ   r   z8make_foreach_pointwise.<locals>.inner.<locals>.<genexpr>  s     2Q1D=2   )r   rI   rR   rS   rT   rV   inplace_foreach_opsr[   r`   r^   r   rd   rm   r   r   has_featureBackendFeatureFOREACHrealizeget_operation_nameregister_operation_listr  )r5  r/  realize_outputsa_list_inputinputbroadcast_inputsgroupsoutputsrk   rj   groupoperation_list
output_indri   r  rW  pw_fns                  rZ   rZ  z%make_foreach_pointwise.<locals>.inner  s   $$**+q0 0ww##**.AA0-/ 	  	E%$/$	 ' 	
?	
'
  	/EedE]3 ''#l2C(CD ''.		/ $C)9$:;&3|,,,2LLN 	@(!V[5(*N G "D66F"D\F&,
# GG''0F0FG#'NN$"))&*C*C*EF#G& //?-	@0 2'2222r\   )r/  zlist[list[TensorBox]]r   )rp  rW  rZ  s   `` rZ   make_foreach_pointwiserq    s    45 4 4l Lr\   c                    | j                         k(  r|rt        |       S | S fd} t        |      |       S )Nc                4    t        j                  |       S )N)	src_dtype)rH   r   )r   r   rt  s    rZ   	_to_dtypezto_dtype.<locals>._to_dtype  s    ||Au	::r\   r0  )r   cloner[  )r   r   copyru  rt  s    `  @rZ   r   r     sC    IEuQx&Q&; B>)5A!DDr\   r   c                   ddl m} |}| j                  } ||t        j                        }t        j
                  |      5   |j                  |  ddd       |j                  }|sJ t        |      }dgt        |      z  }	|j                         D ]  \  \  }
}}g }|D ]k  \  }}||	|<   t        j                  j                  |
t        j                        s:|s=|j                          |j                  |j!                                m |s~t        j                  j#                  |        t%        d |	D              sJ |	S # 1 sw Y   xY w)aI  
    This lowers an invocation of foreach_map
    The way this works is that an arbitrary N-arg func is provided by the user, looped over by the
    polyfill with the same semantics as a foreach op (a loop applying an n-ary function to n args)
    and then traced into a subgraph by dynamo.
    This code allows us to inline the subgraph into the main graph lowering using the PontwiseSubgraphLowering.
    The graph outputs represent the vertically fused sequence of ops, and then register_operation_list
    below registers the buffers as horizontally fuseable in the scheduler.
    r(   )PointwiseSubgraphLowering)root_graph_loweringNc              3  $   K   | ]  }|d u 
 y wr   r   r  s     rZ   r   z_foreach_map.<locals>.<genexpr>
  s     .q}.r_  )subgraph_loweringr{  graph_modulerI   rR   set_graph_handlerrungraph_outputsrm   r   r   ra  rb  rc  rd  rd   re  rf  r  )subgraphri   r   r{  r/  gmpw_subgraphsub_outputsrk  rl  rk   rj   rm  rn  ro  r  s                   rZ   _foreach_mapr    sA    =F			B+BAGGLK	
		[	) ! ! ++K;,Ffs;''G(. <$u$& 	C 
"(GJww""6>+A+AB{ %%f&?&?&AB	C GG++N;< .g....N3! !s   E		Ec                @   |j                   s| j                         j                   rk| j                         r/t        | |      }t        j
                  j                  ||        |S  t        t        j                  j                  d      | |      S t        | |d      S )Nr   Fadd_to_fallback_setTrx  )
is_complexr   r   
empty_liker+   InplaceCopyFallbackr   fallback_handlerprimsconvert_element_typedefaultr   )r   r   dsts      rZ   _convert_element_typer    s    1;;=33::< Qe,C""))#q1J#**22  Au4((r\   r  c                  | j                         }||k(  r|rt        |       S | S d } ||      } ||      }||k7  r* t        t        j                  j
                        | |      S t        t        j                  | |            S )Nc                    | j                   rt        j                  |       j                  S t        j                  |       j                  S r   )is_floating_pointrp   finfobitsiinfor   s    rZ   _get_primitive_bitwidthz1to_dtype_bitcast.<locals>._get_primitive_bitwidth#  s5    "";;u%***;;u%***r\   )	r   rw  r  atenviewr   r:   r/   r   )r   r   rx  x_dtyper  src_bitsdst_bitss          rZ   to_dtype_bitcastr    s~    kkmG%uQx&Q&+ 'w/H&u-H80		0E::))!U344r\   c                &   |j                   s| j                         j                   r`t        j                  t        j
                  j                  t        j                  j                  j                  j                  | |            S t        | |      S r   )r  r   r:   r   r+   ComplexViewrp   rH   r  r  r   r  r)  s     rZ   _view_dtyper  2  se    1;;=33NN!!%)).."5"5";";QF
 	
 Au%%r\   rx  non_blockingc                   t        |      }| j                         |k(  r|rt        |       S | S t        j                  t
        j                  j	                  | ||            S r   )r>   rc   rw  r:   r   r+   
DeviceCopy)r   rk   rx  r  s       rZ   	to_devicer  ;  sO    6"F||~uQx&Q&BMM00FLIJJr\   c                     t        | |d|      S )NTr  )r  )r   rk   r  s      rZ   _device_putr  B  s    QTEEr\   Tc
                h   |xs | j                   }t        |      }
|rt        d|z         }t        d|z   ||       t        |||       |t        |      }t        |
|||rnd||	      }
 t	        | |||      |
      }
t        t        |      r" t	        t        t        |      d|      |
       |
S )z3A pointwise function that maps ops.{name} to inputs
libdevice_N)r0  rJ  rI  rW  rY  r  )r   r   )__name__r5   rE   r[  r  r   r  r   )r   r   r   r   r   r0  rJ  rW  use_libdevice_for_f64rY  rx   fn_libdevices               rZ   register_pointwiser  G  s     #7##D	T	B"<$#67+4!46K	
 (!#8 #.&12M&N#	
3$?5J\PT'
B
	/3	

 	

B ud	
E4  $"7	
 		
 Ir\   c                 .   d} t        d      fd}fd}t        |      t        |t        j                        gfd} t	        t
        j                        |      }t        t        |       r! t	        t        t        |       d      |       |S )z2A pointwise function that maps ops.frexp to inputsfrexpc                      | i |d   S Nr   r   ri   r   r  s     rZ   frexp0zregister_frexp.<locals>.frexp0      d%f%a((r\   c                      | i |d   S Nr(   r   r  s     rZ   frexp1zregister_frexp.<locals>.frexp1  r  r\   rv  c                 0     d   | i | d   | i |fS Nr   r(   r   )ri   r   pw_fnss     rZ   rx   zregister_frexp.<locals>.fn  s.    vay$)&)96!9d+Ef+EEEr\   Nry  )
r5   r[  rp   int32r  r  r  r   r  r   )r   r  r  rx   r  r  s       @@rZ   register_frexpr  {  s    D E)) 	vvU[[AF
F
	




B ud	
E4  $	
 	 Ir\   c                8    t        ||      }t        | |      }|S )NrW  )rq  r   )r   pointwise_lowering_fnrW  rx   s       rZ   register_foreach_pointwiser    s"    
 
  5;	OB	#GR	0BIr\   )r   r   c           
        d }t        |t        t        f      r t        |      |      }t        |t        t        f      r t        |      |      }| ||g}t	        |d   |d   t
        j                        }t        |      D cg c]  \  }}t        |t              s| }}}t        |t        |D cg c]  }||   	 c}       D ]
  \  }}|||<    t        t        |            D ]Y  }t        ||   t        j                        s!t        j                   ||   t#        ||d      j%                                     ||<   [  t'        ||      |d   t)        |d   |      t)        |d   |            S c c}}w c c}w )Nc                 &    t        j                  |  S r   )rH   whereri   s    rZ   rx   zwhere.<locals>.fn  s    yy$r\   r(   r&   ry  r   rv  )r`   r#  r   constant_liker   r   r,  r_   r:   r   r   r   r   r+   r   r0   r   r^   r   r[  r   )	r   r   r  rx   ri   r   rh   r   indicess	            rZ   r  r    so     !eS\"M!Q!eS\"M!Q!Q<DQa.M.U.UE 't_ITQ
1i0HqIGIG.'0JQa0JKL 1Q3t9 Td1gr{{+ ''Qd71:6F6O6O6Q1RSDGT ;>"E:Q$q'5)8DGU+C  J0Js   E9 E96E?c                    t        |       dk(  r$t        | d   t        t        f      rt	        | d    S t        j                  t        | D cg c]  }|j                          c}g       }g }| D ]b  }|j                         }t        |      t        |      k7  st        d t        ||      D              rt        ||      }|j                  |       d |S c c}w )Nr(   r   c              3  ~  K   | ]4  \  }}t         j                  j                  j                  j	                  t        j                  |d       d      xrJ t         j                  j                  j                  j	                  t        j                  |d       d       xs t         j                  j                  j                  j	                  t        j                  |d       d       xrI t         j                  j                  j                  j	                  t        j                  |d       d       7 yw)r(   Tr  N)rI   rR   r  r  r  r   r  r   r   r  s      rZ   r   z$broadcast_tensors.<locals>.<genexpr>  s
     ,
& 1! GG$$..<<At =   GG,,66DDAt E  	 ((22@@At A    ((22@@At A ,
s   D;D=)r   r`   r^   r   r   r   reducer   r   r  r   r  rd   )r/  r   rV   rl  sizess        rZ   r   r     s    
6{aJvay4-@ &),,(//!&#AQAJJL#A2 F G 

u:V$ ,
& E6*',
 )
* q&!Aq12 N9 $Bs   C
c                    | S r   r   r   s    rZ   nopr    s    Hr\   
lift_freshc                   t        | t              sJ |(t        t        j                  | j                              S t        |t
        t        j                  f      r)t        j                  j                  j                  |      nt        d |D              }t        t        | j                               |      }t!        t        |t              s|fn|      }g }t#        | j                               D ]Z  \  }}||v r@t        j                  j                  j%                  t        j&                  |d      d      rJ|j)                  |       \ || j                         k7  rt+        | |      S | S )Nc              3  n   K   | ]-  }t         j                  j                  j                  |       / y wr   )rI   rR   r  evaluate_static_shaper   ds     rZ   r   zsqueeze.<locals>.<genexpr>  s%     J177##99!<Js   35r(   Tr  )r`   r:   r9   r   rb   r   r   r   rI   rR   r  r  r   r   r   r   r    r_   r  r  rd   r  )r   r   dims	new_shaper  ss         rZ   squeezer    s#   a###
{++AFF344 cC,- 	
..s3JcJJ 
 C

-s
3CJsE$:sfDDI!**,'  1I  ..uxx1~d.SQ  "+ajjl!:49AAr\   c                ,    t        t        | |            S r   )rw  r  )r   r   s     rZ   squeeze_copyr    s    C!!r\   c                    t        | |      }t        | t              sJ t        |t              sJ |j                  | _        | S r   )r  r`   r:   rb   r   r   vals      rZ   squeeze_r    s=    
!S/Ca###c9%%%XXAFHr\   c                    t        |       rt        | dt        j                        S t	        d      } t        |t        j                        |       S )NFr   isinfrv  r   	full_likerp   r   r5   r[  r   rx   s     rZ   r  r    ?    qE44	W	B?>"EJJ?BBr\   c                    t        |       rt        | dt        j                        S t	        d      } t        |t        j                        |       S )NFr   isnanrv  r  r  s     rZ   r  r    r  r\   c                f    t        |       rt        |       S t        d      } t        |      |       S )Nceilr   rw  r5   r[  r  s     rZ   r  r  %  s/    qQx	V	B>"a  r\   c                f    t        |       rt        |       S t        d      } t        |      |       S )Nfloorr  r  s     rZ   r  r  -  /    qQx	W	B>"a  r\   c                f    t        |       rt        |       S t        d      } t        |      |       S )Nroundr  r  s     rZ   r  r  5  s/    qQx!!~b!!$$r\   c                f    t        |       rt        |       S t        d      } t        |      |       S )Ntruncr  r  s     rZ   r  r  >  r  r\   c                   ddl m} t        | g      \  } t        | t        j
                        rt        j                  | t        |            S t        | t              sJ t        |t        t        f      sJ t        | j                               t        |      k(  r| S  || j                               st        j                  j                  j                  t!        | j                                     }|dkD  rL ||      sD| j#                  t        j                  j                  j                  t!        |            |z         t        t        j                  | j$                  t        |                  S )Nr   )free_unbacked_symbols)%torch.fx.experimental.symbolic_shapesr  r2  r`   r+   rQ  r0   r   r   r:   r^   r   rI   rR   r  	size_hintrF   
mark_reuserb   )r   r  r  x_size_products       rZ   r  r  F  s   KaS!DQ!R__%  E%L11a###edE]+++QZZ\eEl* .))33M!**,4OP
 A&;E&BLL  **=+?@NR Z&&qvvuU|<==r\   c                    t        |      }|D ]  }d||<   	 | }t        |      D ]  \  }}|dk7  st        ||      } t        ||      S Nr:  )r^   r_   	unsqueezer  )r   shapebroadcast_dimensionsr  broadcast_dimensionr   idxr   s           rZ   broadcast_in_dimr  `  sf    UA3 $!#
$ 	
AA, "Q7!S!A" !Ur\   c                6    t        | |j                               S r   )r  r   )r   r  s     rZ   	expand_asr  n  s    !QZZ\""r\   c                >   t        | j                               t              t              kD  rKt        j                  j
                  gt              t              z
  z  z   t        | t                    } t              t        | j                               k(  sJ t        | j                               }d}t        t                    D ]  }|   dk(  rd}||   |   z  ||<    |r*t        || j                         | j                               S t        d t              D              rt        t        | |            S fd}t        j                   j"                  j%                  t'                    }|dkD  rD| j)                  t        j                   j"                  j%                  t'        |            |z         | j+                         t-        j.                  | j                         | j                         |t        |            S )NFr   Tr   rk   c              3  :   K   | ]  \  }}|d k(  xs |d k(    ywr(   Nr   r  s      rZ   r   zrepeat.<locals>.<genexpr>  s$     
A$!QAFa1f
As   c                   t        |       t              k(  sJ t        |       } t        t                    D ]G  }|   dk7  s|   dk(  rt        j                  j
                  | |<   2t        | |   d|         | |<   I  |       S r  )r   r^   r   r   r  Zeror%   )r(  rh   old_sizerepeatsx_loaders     rZ   rL  zrepeat.<locals>.inner_fn  s    5zS\)))Us7|$ 	IAqzQA;!#$ww||E!H.uQxHQKHE!H	I r\   rM  )r^   r   r   r   r  r  r  r   emptyr   rc   r  r   rw  r  rI   rR   r  r  rF   r  rO  r7   r   )	r   r  new_sizezero_tensorrh   rL  old_size_productr
  r  s	    `     @@rZ   repeatr  s  s   AJJL!H
7|c(m#GGKK=CL3x=$@AHLDN#w<3qzz|,,,,AJJL!HK3w<  /1:?KqkGAJ./
 XQ[[]1<<>JJ

A#gx*@
AAVAx())	 ww''11-2IJ!	GG&&}X'>?CSS	
 }}H||~kkmH~	 r\   c                    t        | t              sJ t        |t        t        f      sJ t        t	        j
                  | j                  |            S r   )r`   r:   r^   r   r<   r   rb   )r   r  s     rZ   r  r    sB     a###edE]+++T[[/00r\   c                    t        | t              sJ t        |t        t        f      sJ t        t	        j
                  | j                  t        |                  S r   )r`   r:   r^   r   r6   r   rb   )r   r  s     rZ   permuter    sF    a###dT5M***[''d<==r\   c           
         t        | t              sJ t        | |d      }t        t        j                  j                  | j                  |||||            S )Nr   clamp)r`   r:   _validate_dimr+   	SliceViewr   rb   )r   r   startendstepr  s         rZ   slice_r    sK    a###
3
"CR\\((eS$e(TUUr\   c                   t        | t              r>t        | j                  t        j                        r| j                  j                         } | j                          t        j                  |       st        d|  d      t        j                  |       \  }}t        j                  |j                  |j                  |D cg c]  }t        j                  |       c}|D cg c]  }t        j                  |       c}t        j                  |xs d            }t        t        j                  ||            S c c}w c c}w )Nzunrealized as_strided(z, ...)r   rb   layout)r`   r:   rb   r+   BaseViewunwrap_viewrd  is_storage_and_layoutr   as_storage_and_layoutFixedLayoutrk   r   r   r  ReinterpretView)r   r   stridestorage_offsetstorage
old_layoutr  
new_layouts           rZ   
as_stridedr,    s    !YJqvvr{{$CFF IIK##A&!$:1#V"DEE2215GZ"&'Qa'"()Qa)^(q)J R''WZHII	 	()s   E&Ec                d    t        | t              sJ t        | |||      j                  | _        | S r   )r`   r:   r,  rb   )r   r   r'  r(  s       rZ   as_strided_r.    s/    a###48==AFHr\   c                4    t        | |||      }t        |      S r   )r,  rw  )r   r   r'  r(  results        rZ   as_strided_copyr1    s    48F=r\   c                    g d} D ]1  }j                  |||j                            z   f       d   d   }3  D cg c]  }|j                          c} fd}t         d   j                               }d   d   |<   t	        j
                   d   j                          d   j                         ||      S c c}w )Nr   r:  c           	       
 t        j                  |    t        j                        }g }g }t	        t                    D ]?  

dk(  r$t        j                  dt        j                        n)t        j                  
   d   t        j                        }t        j                  
   d   t        j                        }t        j                  ||      }t        j                  ||      }
dk(  r|}n*
t              dz
  k(  r|}nt        j                  ||      }|j                  |       t        |       t           
   d   z
        <   |j                  t        j                  |
fdd             B |d   }	t	        t              dz
  dd      D ]  
t        j                  |
   |
   |	      }	! |	S )Nr   r(   c                                S r   r   )rh   idx_loadinputs_loaderss   rZ   <lambda>z1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>  s    -N1-h7 r\           r:  r&   )rH   
index_exprrp   int64r   r   constantgeltand_rd   r^   r#   maskedr  )r   idx_dimmasksmasked_loadsr  r  
start_condend_condmasknext_valrh   r5  r   r/  r6  inputs_rangess             @@rZ   rL  zpointwise_cat.<locals>.inner_fn  s   ..S5;;7s6{#  	A 6 Q,^^M!$4Q$7E 
 ..q!1!!4ekkBC/Jvvgs+HAvc&kAo%!xx
H5LLCyH %Xc]]15Ea5H%HIHSM

75 	D  #Fq("b1 	AyyaQH	 r\   rM  )rd   r   rO  r^   r7   r   rc   r   )r/  r   prev_endr   rL  r  r6  rG  s   ``    @@rZ   pointwise_catrI    s    9;MH )h3<<>#3F(FGH $R() 4::Ccoo':N.` F1I&&()H!"%b)HSMay##%Qi!!#	 k ;s   Cc           	       	
 t        j                               dk(  sJ d       t        j                               dk(  sJ d       | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 t        | j                               k  s!J dt        | j                                       | j                         j                         	j                         
	
f	d}t        j                  | j                         || j                               S )Nr(   expect scales 1 dimexpect zero_points 1 dim<Expecting input to have dtype torch.float32, but got dtype: Expecting axis to be < c                b  	 | 
   f} |       } |      } |      }t        t        j                        \  }}j                  t        j                  k7  r$t	        j
                  |t        j                        }j                  t        j                  k7  r$t	        j
                  |t        j                        }t	        j                  |      }t	        j                  ||z        |z   }t	        j                  |t	        j                  ||            }	t	        j
                  |	      S Nr   )_create_constantsrp   float32r   rH   r   r  
reciprocalr  maximumminimum)r   channel_idxri  scale
zero_pointqminqmax	inv_scaler  clampedaxisr   input_loader	quant_max	quant_minscalesscales_loaderzero_pointszero_points_loaders             rZ   rL  z;quantized_decomposed_quantize_per_channel.<locals>.inner_fn;  s    4ylS!k*'4
&y)5==Q
d<<5==(LL6E+j%++>JNN5)	ii	)*Z7++dCKKc$:;||GU++r\   rM  )r   r   r   rp   rR  r   rR  rO  r7   r   rc   )ri  ra  rc  r]  r`  r_  r   rL  r^  rb  rd  s    `````` @@@rZ   )quantized_decomposed_quantize_per_channelre  !  s?    v !Q&=(==&{##%&!+G-GG+ENN*.??- 
FuGXFYZ- #enn&'' 
!#enn&6"7!89' $$&L&&(M$002, ," !~~	 r\   )	out_dtypec               v  	
 t        j                               dk(  sJ d       t        j                               dk(  sJ d       | j                         |k(  sJ d| d| j                                 t        | j                               k  s!J dt        | j                                       t        j                  | j                         	j                         
j                         	
fd}t        j                  | j                         || j                               S )	Nr(   rK  rL  Expecting input to have dtype , but got dtype: rN  c                   |    f} |       } 
|      } |      }	j                   t        j                  k7  r$t        j                  |t        j                        }j                   t        j                  k7  r$t        j                  |t        j                        }t        j
                  t        j                  |t        j                        |      |z  }t        j                  |      }|S r   )r   rp   rR  rH   r   sub)r   rV  ri  rW  rX  r  r]  r^  rf  ra  rb  rc  rd  s         rZ   rL  z=quantized_decomposed_dequantize_per_channel.<locals>.inner_fnr  s    4ylS!k*'4
<<5==(LL6E-j%--@Jggcll5%--8*EMll3	*
r\   rM  	r   r   r   rp   rR  rO  r7   r   rc   )ri  ra  rc  r]  r`  r_  r   rf  rL  r^  rb  rd  s    ```   ` @@@rZ   +quantized_decomposed_dequantize_per_channelrm  T  s-    v !Q&=(==&{##%&!+G-GG+??% 
(/@AR@ST% #enn&'' 
!#enn&6"7!89' MM	$$&L&&(M$002  !~~	 r\   c                   | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 | j                         fd}t        j                  | j                         t        j                  |t        |      t        |            | j                               S )NrM  c                L    
|       }t        d|z  |t        j                        \  }}t        j                  ||z        |z   }t        t        j                        \  }}t        j
                  t        j                  ||      |      }t        j                  |	      S )N      ?r   )rQ  rp   rR  rH   r  rU  rT  r   )r   rW  rX  ri  r[  r  rY  rZ  r\  r   r^  r_  r`  s            rZ   rL  zBquantized_decomposed_quantize_per_tensor_default.<locals>.inner_fn  s    S! 1%K5==!
	: ii	)*Z7&y)5==Q
d++ckk#t4d;||GU++r\   rW  rX  rM  )r   rp   rR  r   rR  rO  r7   r   rc   r   r  r#  r   r   )ri  rW  rX  r`  r_  r   rL  r^  s      ``` @rZ   0quantized_decomposed_quantize_per_tensor_defaultrr    s     ENN*.??- 
FuGXFYZ- $$&L, !""E%LS_
 ~~ r\   c               l   | j                         |k(  sJ d| d| j                                 t        j                  | j                         fd}t	        j
                  | j                         t        j                  |t        |      t        |            | j                               S )Nrh  ri  c                     |       }t        ||t        j                        \  }}t        j                  t        j
                  |t        j                        |      |z  }t        j
                  |      }|S rP  )rQ  rp   rR  rH   rk  r   )r   rW  rX  ri  r  r^  rf  s        rZ   rL  zDquantized_decomposed_dequantize_per_tensor_default.<locals>.inner_fn  s]    S!-eZu}}Uzggcll5%--8*EMll3	*
r\   rq  rM  )r   rp   rR  rO  r7   r   rc   r   r  r#  r   r   )	ri  rW  rX  r`  r_  r   rf  rL  r^  s	         ` @rZ   2quantized_decomposed_dequantize_per_tensor_defaultru    s     ??% 
(/@AR@ST% MM	$$&L !""E%LS_
 ~~ r\   c                0  	 | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       | j                         j                         j                         		fd}t        j                  | j                         || j                               S )NrM  r   r(   expect scale as scalar tensor"expect zero_point as scalar tensorc                    	|       } t        j                               dk(  rdnd      } t        j                               dk(  rdnd      }j                  t        j                  k7  r$t        j                  |t        j                        }j                  t        j                  k7  r$t        j                  |t        j                        }t        j                  |t        j                  |      z        |z   }t        
t        j                        \  }}t        j                  t        j                  ||      |      }t        j                  |      S )Nr(   r   r   r   )r   r   r   rp   rR  rH   r   r  rS  rQ  rU  rT  )r   ri  _scale_zero_pointr  rY  rZ  r\  r   r^  r_  r`  rW  scale_loaderrX  zero_point_loaders           rZ   rL  zAquantized_decomposed_quantize_per_tensor_tensor.<locals>.inner_fn  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKiiv 667+E&y)5==Q
d++ckk#t4d;||GU++r\   rM  )r   rp   rR  r   rR  r   r   rO  r7   r   rc   )
ri  rW  rX  r`  r_  r   rL  r^  r}  r~  s
    ````` @@@rZ   /quantized_decomposed_quantize_per_tensor_tensorr    sf    ENN*.??- 
FuGXFYZ- u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  $$&L$$&L"..0, , !~~	 r\   c                 	
 t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       | j                         |k(  sJ d| d| j                                 t        j                  | j                         j                         	j                         
	
fd}t        j                  | j                         || j                               S )	Nr   r(   rw  rx  rh  ri  c                V    |       } t        j                               dk(  rdnd      } 
t        j                               dk(  rdnd      }j                  t        j                  k7  r$t        j                  |t        j                        }	j                  t        j                  k7  r$t        j                  |t        j                        }t        j                  t        j                  |t        j                        |      |z  }t        j                  |      }|S )Nr(   rz  r   )r   r   r   rp   rR  rH   r   rk  )r   ri  r{  r|  r  r^  rf  rW  r}  rX  r~  s        rZ   rL  zCquantized_decomposed_dequantize_per_tensor_tensor.<locals>.inner_fn%  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKggcll5%--8+FOll3	*
r\   rM  rl  )ri  rW  rX  r`  r_  r   rf  rL  r^  r}  r~  s    ``   ` @@@rZ   1quantized_decomposed_dequantize_per_tensor_tensorr    sU    u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  ??% 
(/@AR@ST% MM	$$&L$$&L"..0
 
 !~~	 r\   c                   | d   j                         j                  dk(  }|rt        d | D              rp| D ]  }|j                           t        d | D              rt	        t
        j                  g|  \  } } t        t
        j                  j                        | |      S t        |       dk(  rt        | d         S t        | d   |d      }t        | dt        j                  i}| D cg c]  }t        ||       } }ddd fd	t!        fd
| D              }dfdt"        j$                  rt'        | |      S |r)t)        t*        j,                  j/                  | |            S fdd}dddt        |       |k  s1t        |       t"        j0                  k  rt        fd| D              rt        fdt2        j4                  j6                  D              }	t!        fd| D              xr |	}
t        fd| D              xr t!        fd| D               }|
s|r|st'        | |      S t)        t*        j,                  j/                  | |            S c c}w )Nr   cpuc              3  |   K   | ]4  }|j                         t        j                  t        j                  fv  6 y wr   )r   rp   int8uint8r   ri  s     rZ   r   zcat.<locals>.<genexpr><  s.      ;@ejj%++66s   :<c              3  T   K   | ]   }t        |j                               d k(   " yw)r   N)r   r   r  s     rZ   r   zcat.<locals>.<genexpr>C  s!     >es5>>#$)>s   &(r(   r   c                   t        | t              rJt        | j                  t        j                        r| j                  j                         S | j                  S t        | t        j                        r| j                  S | S r   )r`   r:   rb   r+   r!  r"  
StorageBoxr   s    rZ   unwrap_tensorzcat.<locals>.unwrap_tensorP  sV    a#!&&"++.vv))++vva'66Mr\   c                    t        | t        j                        xr$ t        | j                  t        j                        S r   )r`   r+   ComputedBufferrb   r8   )rl   s    rZ   is_reductionzcat.<locals>.is_reduction\  s)    !R../TJqvvr||4TTr\   c                    t        | t        t        j                  f      r  |             S  |       xs> t        | t        j                        xr" t        fd| j                         D              S )Nc              3  h   K   | ])  } t         j                  j                  |             + y wr   )rI   rR   
get_buffer)r   readcan_fuse_reductions     rZ   r   z2cat.<locals>.can_fuse_reduction.<locals>.<genexpr>e  s-       #177#5#5d#;<s   /2)r`   r:   r+   r  r7   r  get_read_names)rl   r  r  r  s    rZ   r  zcat.<locals>.can_fuse_reduction_  sh    a)R]]34%mA&677O !R\\*  ,,. 	
r\   c              3  .   K   | ]  } |        y wr   r   r   rl   r  s     rZ   r   zcat.<locals>.<genexpr>l  s     Ba.q1B   c                6   t        j                  |       r:t        j                  | d      \  }}t         j                  j	                  |       S t        | t        t         j                  f      r  |             S t        | t         j                        ryy)NF)freezeT)	r+   r#  r$  ConcatKernelcan_realize_into_without_copyr`   r:   r  r7   )r   r)  r   should_lower_cat_inputr  s      rZ   r  z#cat.<locals>.should_lower_cat_inputn  sx     ##A&11!EBJGQDDWMMMa)R]]34)-*:;;a&r\   c                H   t        | t        t        j                  f      r  |             S t        | t        j                        sy| j                         j                  }| j                         D ]*  }| t        j                  j                  |            z  }, |S r  )r`   r:   r+   r  r7   inner_fn_opcountnum_opsr  rI   rR   r  )r   countr  op_countr  s      rZ   r  zcat.<locals>.op_count  s    a)R]]34M!,-- !R\\*""$,,$$& 	8DXagg00677E	8 r\   r   r&   c                n    | t         j                  j                  t         j                  j                  fv S r   )r  catr  constant_pad_ndrU   s    rZ   additional_pointwise_opsz%cat.<locals>.additional_pointwise_ops  s(    dhh&&(<(<(D(DEEEr\   c              3  4   K   | ]  } |      k    y wr   r   )r   rl   MAX_SIMPLE_OP_COUNTr  s     rZ   r   zcat.<locals>.<genexpr>  s     Cq22Cs   c              3  6   K   | ]  }t        |        y wr   )rA   )r   user  s     rZ   r   zcat.<locals>.<genexpr>  s!      
 S":;
r   c              3  .   K   | ]  } |        y wr   r   r   r   r  s     rZ   r   zcat.<locals>.<genexpr>  s     >&s+>r  c              3  .   K   | ]  } |        y wr   r   r  s     rZ   r   zcat.<locals>.<genexpr>  s      "
,/"3'"
r  c              3  .   K   | ]  } |        y wr   r   r  s     rZ   r   zcat.<locals>.<genexpr>  s     <(+<r  )r   zUnion[TensorBox, ir.StorageBox]return	ir.IRNode)r  r   rU   torch._ops.OpOverload)rc   rP  r  rd  require_channels_lastr  r  r  r  r   rw  r  r   r   r,  r   r  r)   force_pointwise_catrI  r:   r+   r  r   max_pointwise_cat_inputsrI   rS   rT   )r/  r   
cpu_deviceri  r   r   r   fusable_reductionMAX_COMPLEX_POINTWISE_CATpointwise_usesfuse_pointwise_usehorizontal_fuse_catr  r  r  r  r  r  r  s               @@@@@@@rZ   r  r  9  sI   %%',,5Jc DJ 
  	EMMO	>v>>-dhh@@IFA1 0 01&#>>
6{aVAY
q	3
*C	%D%L%LE /55shsE"5F5
U

 B6BB !!VS)) //<==$ !"F 6{//	V77	7CFCC 
~~++
 
 >v>>Q> 	 " "
39"
 
 =<V<<< 	 "5>O --R__++FC899K 6s   5I1c                  
 | j                         t              t              t              t        k7  fd       t        j
                  j                  j                  t        j                  |d            }|r\t        j
                  j                  j                  t        j
                  j                  j                     |z            d      }n[t        j
                  j                  j                  t        j
                  j                  j                        |z
        d      }d
|r| df
nd|f
t              D cg c]  \  }}|fvs| }}}|j                  |       
fd}	t        t        j                   j#                  | ||	            S c c}}w )N)r   rankc                     d  d S )Nz(diagonal dimensions cannot be identical z, r   )dim1dim2s   rZ   r7  zdiagonal.<locals>.<lambda>  s     HbQUPVW r\   r   )r   r   c                    | d   }dgt        	      z  }d}t              D ]1  }|k(  r|d   z   ||<   |k(  r|d   z   ||<   %| |   ||<   |dz  }3 |t        	      dz
  k(  sJ |S )Nr:  r   r(   r&   )r   r   )
r   diag_idxoriginal_idxcur_dimr  base_idxr  r  num_dimsoriginal_shapes
        rZ   	reindexerzdiagonal.<locals>.reindexer  s    r7sS00x 	ADy"*Xa["8Qd"*Xa["8Q"%g,Q1	 #n-1111r\   )r   r   r   r   rI   rR   r  r  r   Ltevaluate_maxevaluate_minr_   rd   r:   r+   GenericViewr   )ri  offsetr  r  offset_negative	diag_sizerh   r  r  r  r  r  r  s     ``      @@@rZ   diagonalr    s   ^^%N>"H84D84D	W gg&&44UXXfa5HIOGG$$11GG))t$v-~d/C 	
	 GG$$11GG))t$nT&:V&C 	
	 HGQ<v;$^4N41a$8MQNEN	LL   R^^**5%CDD' Os    F:/F:c                0    t        t        | |||            S r   )rw  r  )ri  r  r  r  s       rZ   diagonal_copyr    s    %t455r\   c                P    t        |       }t        ||||      }t        ||       |S r   )rw  r  	mutate_to)ri  srcr  r  r  r  rV   s          rZ   diagonal_scatterr    s*    5\FffdD1FfcMr\   c           	         t        j                  || j                         |         }t        t	        | |||dz         |      S r  )r<   handle_negative_indexr   r  r  )r   r   r   s      rZ   selectr    s;    

$
$S!**,s*;
<C6!S#sQw/55r\   c           
     r   t        | |d      }|}t        |t        t        f      s`| j	                         |   }t
        j                  j                  j                  t        ||z   dz
  |            }|g|z  }||dz
  |z  z
  |d<   g }d}|D ](  }||z   }	|j                  t        | |||	d             |	}* |S )Nr   r(   r:  Fr  )r  r`   r^   r   r   rI   rR   r  r  r"   rd   r  )
r   r  r   sizes_x_sizechunksr0  r  r   r  s
             rZ   splitr     s    
3
"CF edE]+c"!!77Ve^a'/
 6!vzU22r
 FE dl 	fQUCu=> Mr\   c                    t        | ||      S r   )r  )r   r  r   s      rZ   split_with_sizesr    s    E3r\   c                    t        | |d      }t        j                  j                  j	                  | j                         |         }t        |      D cg c]  }t        | ||       }}|S c c}w r  )r  rI   rR   r  r  r   r   r  )r   r   r  rh   r0  s        rZ   unbindr  "  s_    
3
"CWW33AJJL4EFF).v7AfQQ7F7M 8s   A-c                >   | j                         }t        |      }t        ||      |dk(  rt        t	        | d      |      S |   }t
        j                  j                  }|j                  ||       |j                  d       t        ||z
        dz   }|j                  |      dkD  r-| j                  |j                  t        ||z  |                   g |d  ||dz   d  |}	fd}
t        t        j                   j#                  | |	|
            S )Nr   )r  r(   c                D    | d   |    z  z   }g | d  || dz   d S )Nr:  r(   r   )r   dim_idxr   r  s     rZ   r  zunfold.<locals>.reindexer>  s?    b'CHtO+8Tc8G8c#'B&788r\   )r   r   r   r  r  rI   rR   r  	guard_leqguard_ltr"   r  r  r!   r:   r+   r  r   )r   	dimensionr   r  r  ndimdim_sizer  new_dim_sizeout_sizer  r   s      `       @rZ   unfoldr  *  s   JJLEu:D
4
+Cqyi1o400SzHwwHtX&aHtOT2Q6L(#a'	X''t0CX(NOPDtDlDU379-=DtDH9 R^^**1h	BCCr\   c                    t        | |d      }t        | j                               }|j                  |t        j
                  j                         t        | |      S r  )r  r^   r   insertr   r  r  r  )r   r   r  s      rZ   r  r  E  sE    
3
"CQZZ\"IS%''++&9r\   c                    t        | |      }t        | t              sJ t        |t              sJ |j                  | _        | S r   )r  r`   r:   rb   r  s      rZ   
unsqueeze_r  M  s>    
As
Ca###c9%%%XXAFHr\   c                   t         j                  j                  j                  j	                  t        j                  |            }t        | j                               }|dk  r|||z   z  }d|cxk  r	||z   k  sJ  J |S r  )	rI   rR   r  r  r  r   sympifyr   r   )r   r   r  r  s       rZ   r  r  V  sr    
''


$
$
2
25==3E
FCqzz|D
Qwtf}#dVm#####Jr\   r:  c                    t        | |d      }t        j                  j                  j	                  | j                         |         dz  }t        | |d|      }t        | |||dz        }t        |t        |            S )Nr   r&   )	r  rI   rR   r  r  r   r  rN  sigmoid)r   r   new_lenr   r  s        rZ   glur  _  so    
3
"Cgg44QZZ\#5FG1LGq#q'"Aq#w!,Aq'!*r\   c                N     |rt         j                           fd}d|_        |S )Nc                 x    d }t        j                  |t        j                  j                  g| i |      S )Nc                d    t        | t        j                        rt        j                  |       S | S r   )r`   r+   r2   r:   r   r   s    rZ   wrap_tensorsz7fallback_handler.<locals>.handler.<locals>.wrap_tensorsn  s%    *4Q		*B9##A&IIr\   )pytreetree_mapr+   FallbackKernelr   )ri   r   r  kernels      rZ   handlerz!fallback_handler.<locals>.handlerm  s<    	J "++226KDKFK
 	
r\   T)r  r   _is_fallback_handler)r   r  r  s   `  rZ   r  r  i  s&    f
 $(G Nr\   c                 .    t        j                  d       y )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnr   r\   rZ   _warn_complex_not_supportedr  {  s    MMtr\   c                    | j                         rv|ri|j                  t        j                  j                  j
                  j                  t        j                  j                  j                  j                  fv ryt                y| j                  t        j                  k(  r[|sy|j                  t        j
                  j                  t        j                  j                  fv xs t        |j                         S y)z0Do not support reading or writing to this tensorFT)r  rV   rp   rH   r  r  r   r  r  r  r  float8_e8m0fnur  rB   rl   parentrX   s      rZ   unsupported_input_tensorr    s    ||~fmmIINN%%IIOO0088(
 
 #%ww%&&&
 KK		   $
 t{{#
 	
 r\   c                X    t        | |      ry| j                  xr t        j                  S )z2Do not support writing tensor but can read from itT)r  is_cpur)   disable_cpp_codegenr	  s      rZ   unsupported_output_tensorr    s$    6*882222r\   c                0   | j                   t        j                  j                  u ry| j                   t        j                  j                  u ryd }t        j                  | j                  i | j                  D ]  } ||| d      s y  || | d      S )NFc                F   t        | t        j                  j                        syd| j                  vryt        j                  | j                  d         D ]H  }t        |t        j                  j                        s(|rt        |||       s8 yt        |||       sH y y)NFr  T)r`   rp   fxNoderT  r  tree_leaves_subclasses
FakeTensorr  r  )rX   r
  	is_outputrT  s       rZ   check_skip_conditionzCfallback_node_due_to_unsupported_type.<locals>.check_skip_condition  s    $.		!&&tyy'78 		 DdE$5$5$@$@A,T64@+D&$?		  r\   )r  T)	rV   r  view_as_complexr  lift_fresh_copyr  arg_tree_leavesri   r   )rX   allow_cpu_inputsr  r   s       rZ   %fallback_node_due_to_unsupported_typer    s    {{d**222 {{d**222* %%tyy@DKK@ TU;  dd;;r\   c                *   | t         vs|s
J d|         |rt        t        j                  d            rt	        | g      rt
        j                  r&| t        j                  j                  j                  v si|sgt        j                  j
                  j                  r4dt        j                  j
                  _        t        j                  d       t        d|  d      fd}t!        | t        j"                  j$                        r*| j'                         D ]  }t)        | |      } ||        y t!        | t        j"                  j*                  t        j"                  j,                  f      r	 ||        y t/        d|  d	t1        |              )
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                n    t        |        t        |         t        | d       t        |             S Nry  )r   r   r  r  )op_overloadlayout_constraints    rZ   register_fallbackz(make_fallback.<locals>.register_fallback  s;    !+.(!+/@AG $G[)
 	
r\   zUnsupported fallback z with type )r-   r   osgetenvr.   r)   fallback_randomrp   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningrv   r`   rz   r   r   r   r{   HigherOrderOperatorRuntimeErrorrP  )rU   r#  r  override_decompr$  olr"  s    `     rZ   make_fallbackr3    sQ   ^# 
4RD96 	4!t$ ""emm::OOO ==//38EMM  0KKH RD !f f
 	

 "ejj112,,. 	+B!"b/Kk*	+ 
B..

0N0NO	P"22$k$r(LMMr\   c                T    d}| D ]  }||z  }	 t        |t        j                        S )z
    TorchInductor offset calculation differs from PyTorch eager offset
    calculation for random ops (tl.rand vs torch.rand). In future, we should
    strive for same impl for tl.rand and torch.rand.
    r(   r   tensorrp   r:  )r  numelr  s      rZ   philox_rand_offsetr8     s2     E 	%u{{++r\   c                B  	
 t        j                  || t         j                  j                  |             j	                         
|j                         |j                         		
fd}t        j                  ||t        |             }t        |       }||fS )Nc                   t        j                   g       t        j                        }t        j                   g       t        j                        }t        j                  t        j
                   |       t        j                        |      }t        j                  ||      }t        j                  |      S r   )rH   r   rp   r  r   r9  rand)	r(  seed_index_exproffset_index_exprrand_index_exprr0  r   offset_loader
random_posseed_loaders	        rZ   rL  zphilox_rand.<locals>.inner_fn  s     ,,{2DLLr):EKKH''NN:e,ekk:<M
 
 ||FE**r\   rM  )
r+   r%  FlexibleLayoutcontiguous_stridesmake_indexerrO  r7   r   r^   r8  )r   seedr  r'  rk   r   rL  random_values_nodeoffset_noder?  r@  rA  s        `   @@@rZ   philox_randrH    s     
,,T2	
 ln  ""$K&&(M+ #))Dz	 %T*K{**r\   c           	         t         j                  r\t        j                  t        j
                  t        j                  j                  t        j                  j                  | ||            S t        d      )Nz&should be handled in replace_random.py)r)   r'  r  r  r:   r   r+   r  r  native_dropoutr  rv   )r   ptrains      rZ   rJ  rJ  3  sY    $$T%8%8%@%@!QN
 	

 EFFr\   c                x   t         j                  s-| j                         t        j                  d      k(  sJ d       | j                          t        |      dk(  st        |d   t              rt        j                  j                  nt        j                  j                  }t        j                  || g|  | S )Nr  Tthis should be handled in decomps unless config.fallback_random or the device is CPUr   )r)   r'  rc   rp   rk   rd  r   r`   r#  r  
bernoulli_Tensorr+   InplaceBernoulliFallback)r   ri   r"  s      rZ   rO  rO  >  s    !!Q\\^u||E7J%J ^J IIK t9>ZQ7 	__## 
 Q66Hr\   c                    t         j                  s-| j                         t        j                  d      k(  sJ d       t        t        |       g| S )Nr  rN  )r)   r'  rc   rp   rk   rO  rw  )r   ri   s     rZ   bernoulli_prS  M  sG    !!Q\\^u||E7J%J ^J eAh&&&r\   c                    t         r   rv   r   s    rZ   _foobarrW  V  s    
r\   c                .    t         j                  d       y )Nz1using triton random, expect difference from eager)r-  info)salts    rZ   _warn_triton_randomr[  [  s    HH@Ar\   c                 J    t        t        j                  j                         y r   )r[  rI   rR   creation_timer   r\   rZ   warn_triton_randomr^  `  s    --.r\   c                     |j                  dd       t        | i |S t        j                  r|j	                  dd        t        | i |S t        d      N	generatorz-should have been handled in replace_random.py)rU  fallback_rand_generatorr)   r'  popfallback_rand_defaultrv   ri   r   s     rZ   r;  r;  l  sV    zz+t$0&777			

;%$d5f55
H
IIr\   c                     |j                  dd       t        | i |S t        j                  r|j	                  dd        t        | i |S t        d      r`  )rU  fallback_randn_generatorr)   r'  rc  fallback_randn_defaultrv   re  s     rZ   randnri  v  sV    zz+t$0'888			

;%%t6v66
H
IIr\   c                l    t        j                  |      }t         j                  j                  | |      S r   )r+   get_stride_orderExternKernelrequire_stride_order)input_tensorr'  stride_orders      rZ   inductor_force_stride_orderrp    s)    &&v.L??//lKKr\   c                    t        d      )Nz.should be handled in fuse_seed_creation_pass()rU  )rk   s    rZ   inductor_seedrr    s    
I
JJr\   c                z    t                t        j                  t        j                  | t        |                  S r   )r^  r:   r   r+   RandomSeedsr>   )r  rk   s     rZ   inductor_seedsru    s)    BNN5-2GHIIr\   c                |      fd}t        j                   j                          j                         |g       S )Nc                L    t        j                  j                               S r   )rH   	load_seedget_name)r   r(  seedss    rZ   rL  z&inductor_lookup_seed.<locals>.inner_fn  s    }}U^^-u55r\   rM  )r7   r   rc   r   )rz  r(  rL  s   `` rZ   inductor_lookup_seedr{    s9    6 !oo	 r\   r  c                 	 t         j                  rJ dv sJ g | } t        j                  }|j	                         }t        j                  ||| t
        j                  j                  |       |      j                         |j                         		fd}t        j                  |||g |       }|j                          |S )N)r;  ri  r|  c                     t        t               g       t        j                   |       t        j                              S r   )r   rH   r9  rp   r  )r(  moder@  rA  s    rZ   rL  z!inductor_random.<locals>.inner_fn  s7    !wsD!ONN:e,ekk:
 	
r\   rM  )r)   r'  rp   rR  r.  r+   r%  rB  rC  rD  rO  r7   r   rd  )
r   rE  r  r  r   rk   rL  r0  r@  rA  s
     `     @@rZ   inductor_randomr    s    %%%%$$$$T7DMME%%'FtR..AA$GPVln  ""$K
 w	F NNMr\   c               f   	 t         j                  rJ g |}t        j                  }|j	                         }t        j                  |||t
        j                  j                  |      |      j                         |j                         	 	fd}t        j                  |||g |      S )Nr|  c           	        t        j                   g       t        j                   |       t        j                        t        j                  t        j
                        t        j                  t        j
                              S r   )rH   	randint64r9  rp   r  r:  )r(  highlowr@  rA  s    rZ   rL  z"inductor_randint.<locals>.inner_fn  sV    }}ONN:e,ekk:NN3,NN4-	
 	
r\   rM  )r)   r'  rp   r:  r.  r+   r%  rB  rC  rD  rO  r7   r   )
r  r  r   rE  r  r   rk   rL  r@  rA  s
   ``      @@rZ   inductor_randintr    s     %%%%T7DKKE%%'FtR..AA$GPVln  ""$K
 w	 r\   c                    | j                         | j                         d   | j                         d   | j                         d   z  | j                         d   fS Nr:  r   )ry  r   
get_stridetbs    rZ   _boundaries_helperr    sO    

b
a2==?1--
	 r\   c                H    | j                         | j                         d   fS r  )ry  r  r  s    rZ   _sorter_helperr    s    ;;="--/"---r\   	out_int32rightsidesorterc               &   	
 d } |       r ||      r
9 |      s1 t        t        j                  j                  d       |||      S ||dk(  rd|rt        j
                  nt        j                  	|j                         
 j                          j                          t         j                               dk(  r
	 
fd}n		 
fd	}|j                         }t        j                  |	||j                  
      S )Nc                ^    t         j                  j                  | t        j                        S r   )rI   rR   ra  rb  	BUCKETIZEr  s    rZ   r7  zsearchsorted.<locals>.<lambda>  s     AGG$7$7
N$$% r\   Fr  r  r  Tr(   c           
          |       }t        j                  |t              dd n
t              d       S d      S )Nr   r  sorter_indicesrH   	bucketizer  r  )r   r  index_dtyper  sorted_sequencer  values_loaders     rZ   rL  zsearchsorted.<locals>.inner_fn	  sV    $C=="?3%~t>&3I'-~t  <= r\   c                             }d fd}t        j                  |t               |      d n
t              d       S  |            S )Nc                    | j                         }t        j                  t        j                  t
        j                  d t        |d d d d       D                    S )Nc              3  ,   K   | ]  \  }}||z    y wr   r   )r   r  rh   s      rZ   r   zNsearchsorted.<locals>.inner_fn.<locals>.get_flattened_index.<locals>.<genexpr>*	  s     &UAq1u&U   r:  )r  rH   r9  r   r  operatorr   r   )r  stridesr   r  s     rZ   get_flattened_indexz;searchsorted.<locals>.inner_fn.<locals>.get_flattened_index&	  sU    --/~~$$ &UWSb\3sPR89T&U  	 r\   r  )r  r:   r  )r   r  r  r  r  r  r  r  s   `  rZ   rL  zsearchsorted.<locals>.inner_fn!	  sh    $C =="?3#O4%~t>&3I'-~t  <Ov;V r\   rM  )r  r  searchsortedrP  rp   r  r:  rO  rd  r   r   rc   r7   r   r  )r  selfr  r  r  r  validate_bucketizerL  rk   r  r  s   `  ` `   @@rZ   r  r    s    /!$''9&'AT 1 1 8 8eT
 	
 DGO!*%++K$$&M 
?##%&!+
	 
		 	0 __Fzz	 r\   r  r  c               ^   t        j                               dk(  sJ t        j                  j	                  | t
        j                        r.t        j                  j	                  t
        j                        s/ t        t        j                  j                  d      | |      S j                          | j                         }| j                         |rt        j                  nt        j                   fd}t#        j$                  ||| j                               S )Nr(   Fr  r  c                \     |       }t        j                  |t              d      }|S r  )rH   r  r  )r(  r  r  
boundariesr  r^  r  s      rZ   rL  zbucketize.<locals>.inner_fn^	  s6    5!--z*
 r\   rM  )r   r   rI   rR   ra  rb  r  r  r  r  rP  rd  rc   rO  rp   r  r:  r7   r   )ri  r  r  r  rk   rL  r  r^  s    ` `  @@rZ   r  r  B	  s     z""$%*** 	
E>#;#;<GG
N,D,DEQ 5 55Q:%
 	
 F$$&L!*%++K
 ~~	 r\   c                    t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS r   )r  tree_map_onlyr+   r2   rl  require_stride1r   ri   r   s      rZ   require_denser  r	  s;    ''
		2??22T6NLD& <r\   c                    t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS r   )r  r  r+   r2   rl  require_contiguousr  s      rZ   r  r  y	  s;    ''
		2??55f~LD& <r\   c                    t        j                  t        j                  t        j                  j
                  ||f      \  }}||fS r   )r  r  r+   r2   rl  r  r  s      rZ   r  r  	  s;    ''
		2??884.LD& <r\   c           
         fdt        fdt        | |      D              } |j                         D ci c]  \  }}| |||          }}}| |fS c c}}w )Nc           	        t        | t        j                        rm|j                         D cg c]4  }t        |t        j
                        r|j                  j                  n|6 }}t        j                  j                  | |      S t        | t              r,| j                         D ci c]  }| | |   ||          c}S t        | t        t        f      r$ t        |       fdt        | |      D              S | S c c}w c c}w )Nc              3  6   K   | ]  \  }} ||        y wr   r   )r   r   f_aapply_constraints      rZ   r   zFconstrain_to_fake_tensors.<locals>.apply_constraint.<locals>.<genexpr>	  s"      -5a C(r   )r`   r+   r2   r'  rp   SymIntrX   exprrl  require_exact_stridesr   r  r   r^   rP  r   )r   fake_argr  meta_stride_exprkeyr  s        rZ   r  z3constrain_to_fake_tensors.<locals>.apply_constraint	  s    c299% "*   *!U\\:A    ??88>NOOc4 JM((*CF%c#h>>  eT]+49 9<S(9K   
 s   9D+Dc              3  6   K   | ]  \  }} ||        y wr   r   )r   r   r  r  s      rZ   r   z,constrain_to_fake_tensors.<locals>.<genexpr>	  s"      ,9Ch'r   )r   r   r   )ri   r   	fake_argsfake_kwargsr   r   r  s         @rZ   constrain_to_fake_tensorsr  	  sj    "  =@y=Q D BHPAa!![^44PFP< Qs   Ac           
         fdt        fdt        || j                        D              }|j                         D ci c]  \  }}| || j                  |          }}}||fS c c}}w )Nc           	        t        | t        j                        rst        j                  |j                  d   j                         t        j                  j                  j                        }t        j                  j                  | |      S t        | t              r,| j                         D ci c]  }| | |   ||          c}S | S c c}w Nr  )r`   r+   r2   rk  rT  r'  rI   rR   r  r  rl  rm  r   r  )r   fx_argro  r  r  s       rZ   r  z1constrain_to_fx_strides.<locals>.apply_constraint	  s    c299%..E"))+QWW-=-=-G-GL ??77\JJc4 LOHHJWSC)#c(F3K@@WW
 Xs   1Cc              3  6   K   | ]  \  }} ||        y wr   r   )r   r   r  r  s      rZ   r   z*constrain_to_fx_strides.<locals>.<genexpr>	  s"      *5#vf%r   )r   r   ri   r   r   fx_noderi   r   r   r   r  s        @rZ   rt   rt   	  ss      9<T7<<9P D EKLLNSDAqa!!W^^A%677SFS< Ts   !A*c                      fdt        fdt        t        | j                              D              }|j	                         D ci c]  \  }}| d| j
                  |          }}}||fS c c}}w )Nc           
     
   t        |t        j                        s|S |j                  d   }|j	                         D cg c]4  }t        |t
        j                        r|j                  j                  n|6 }}t        j                  |j	                               }|r<|d   dk7  r4t        t        t        t        |j                                                 }j                  t         j"                  j$                  k(  r| dv rt        |      dk(  sJ d}|j&                  s t        j(                  j+                  ||      S dj                  t
        j,                  j                   j.                  j$                  k(  xr | dk(  }t        |t0              sJ t        |j                               d	vr|S t        j2                  |      r3t        j4                  t        j(                  j7                  |      |      S t        |t              rY|j9                         It        j2                  |      r3t        j4                  t        j(                  j7                  |      |      S |rt        |j                               }g }	|j9                         }
t        t        |j                               d
z
        D ]q  }t:        j<                  j>                  jA                  ||   d      s1|
3t:        j<                  j>                  jA                  |
|   d      sa|	jC                  |       s dgt        |      z  }d
|d<   d
}t        t        |      dz
  dd      D ]  }||d
z      dk7  r|||d
z      z  }||	v r9t:        j<                  j>                  jA                  ||d
z      z  d      rd||<   Vt:        j<                  j>                  jA                  |z  d      stE        |      z  }|||<    t        j(                  jG                  ||      S t        j2                  |      r3t        j4                  t        j(                  j7                  |      |      S t        |t              rY|j9                         It        j2                  |      r3t        j4                  t        j(                  j7                  |      |      S fd}t        |jH                  t        jJ                        rQ ||      sI ||jM                               r3t        j4                  t        j(                  j7                  |      |      S t        j(                  j+                  ||      S c c}w )Nr  r:  r   )r   r   r   )r   r(   r&   r   r   r   r   r   r(   r&   c                    t         j                  j                  j                  | j	                         d         z  dk(  S r  )rI   rR   r  r  r   )r   	ALIGNMENTs    rZ   
is_alignedz=sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned$
  s4    GG$$..qzz|B/?@9LQRRRr\   )'r`   r+   r2   rT  r'  rp   r  rX   r  rk  r^   r  r   r   r   rV   r  0_scaled_dot_product_efficient_attention_backwardr  is_cudarl  rm  rH   '_scaled_dot_product_efficient_attentionr:   is_aligned_realized_tensortry_match_insignificant_stridesrealize_inputmaybe_get_striderI   rR   r  statically_known_equalsrd   r=   r  rb   r!  r"  )r   r   r  meta_valr  r  ro  effn_attn_fwd_biasr  expanded_dimsmaybe_striderh   out_stridesr'  r  r  r  s                  @rZ   r  z)sdpa_constraint.<locals>.apply_constraint	  s   #ryy)J;;u%GOGX
BC:a6AFFKKA=
 
 **8??+<=L,1s3<<>/B)C DEL NNDDLLMv|$)))
 (L??77\JJ 	 NNyy~~EEMMN q 	 #y)))s||~f,J((i855--c24D 
 sF#$$&2--c9=55--c24D  CLLN+HM //1L3s||~.23 ,77##;;<LQ<OQRS ,((@@aRST!((+, $X.KKOF3x=1,b"5 (q1u%*#hq1uo5F %ww''??#AE*Y6 *+A ww''??@RTUV$VY7)CF!'A!($ ??88kJJ((i855--c24D 
 sF#$$&2--c9=55--c24D 	S chh,c?coo/0==55c:<L  33CFFi
s    9U%c              3  >   K   | ]  \  }\  }} |||        y wr   r   )r   r   r   r  r  s       rZ   r   z"sdpa_constraint.<locals>.<genexpr>0
  s*      C#v 	c6*s   r:  )r   r_   r   ri   r   r   r  s   `    @rZ   sdpa_constraintr  	  s}    yGv  "+Cgll,C"D D IOW1a!"a):;;WFW< Xs   "A5)r  c                   |}| j                         |j                         k7  rt        || j                               }| j                         |j                         k7  rt        || j                               }| j	                         |j	                         k7  r%t        || j	                               }t        |      S t        |      S r   )rc   r  r   r   r   r  rw  )r  r  r  r   rf   s        rZ   rx  rx    s    ACNN,,a*+~~3==?*Q()}}#,,.(Q(Sz8Or\   )memory_formatc          	         t        j                  | j                         | j                         | j	                         t        | j                                     S NrM  )r7   r   rc   r   rO  r^   r   )r   r  s     rZ   rw  rw    s@     ||~kkmAJJL!	 r\   c                   g }t        | t              rt        | j                  t        j                        rw| j                  } t        | t        j                        rF|j                  | j                                | j                  } t        | t        j                        rFt        |       } t        |       } |r;| j                  } |d d d   D ]  }t        j                  | |      }  t        |       } | S )Nr:  r  )r`   r:   rb   r+   r&  rd   
get_layoutrw  )r   reinterpret_view_layoutsr   s      rZ   clone_preserve_reinterpret_viewr    s    !!YJqvvr7I7I$JFFB../$++ALLN;A B../ aLaAFF.tt4 	:F""&9A	:aLHr\   r  c               \    fd}t        j                  t        |      || g      S )Nc                D    t        j                  | d   z  z         S )Nr   r   rH   r9  )r(  r   r  r  s    rZ   rx   ziota.<locals>.fnA  s!    ~~dU1Xo5UCCr\   rM  )r7   r   r>   )lengthr  r  r   rk   requires_gradrx   s    ```   rZ   iotar  7  s1    D V$x	 r\   c           	     
   | j                         |j                         k(  sJ | j                         t        | d      t        j                  j
                  j                  t        j                  d            r| j                            z   t        j                  j
                  j                  d       t        j                  j
                  j                  | j                                   t        t        |      | j                               }|j                         fd}t        j                  | j!                         | j                         |t#        | j                                     S )Nr   c           	         t        j                  t        j                  t        j                  |    t        j
                        t        j                  t        j
                               |        |             S r   )rH   r  eqr9  rp   r  )r   r   r(  
src_loaderr  s    rZ   rL  z select_scatter.<locals>.inner_fnX  sX    yyFFs3x5uekk2 sOSM
 	
r\   rM  )r   rO  r  rI   rR   r  r  r   r  r   r  r  r  r  r7   r   rc   r^   )r   r  r   r(  rL  r  r  s     `` @@rZ   select_scatterr  L  s   ;;=CMMO+++}}H
3
"Cww%%ehhua&89

S))GGq%(GGeQZZ\#%67
3$ajjl
3C"J
 ||~kkmAJJL!	 r\   c           	     L   	
  j                         |j                         k(  sJ  j                         
t         d       j                            t        j
                  j                         \  t         j                               }t        z
  dz
  z         |<   t        ||      }|j                         		 
fd}t        j                   j                          j                         |t         j                                     S )Nr   r(   c                   	dk(  rk(  r
dk(  r |       S t        j                  |    t        j                        }t	        |       t        |    	z
  
      <   g }	dk7  rZ|j                  t        j                  |t        j                  t        j                  	      t        j                                     k7  rZ|j                  t        j                  |t        j                  t        j                        t        j                                     
dk7  rz|j                  t        j                  t        j                  t        |    	z
  d
      t        j                        t        j                  dt        j                                     |sJ t        j                  t         j                   |      }t        j"                  |fdt%              rdnd      }t        j&                  || |             S )Nr   r(   c                             S r   r   )src_idxr  s   rZ   r7  z1slice_scatter.<locals>.inner_fn.<locals>.<lambda>  s    Jw' r\   r8  )rH   r9  rp   r:  r^   r"   rd   r<  r   r  r=  r  r%   r;  r   r  r>  r?  r   r  )r   r@  rE  src_valr  r   r  r  r  r  r  r   r  s       @rZ   rL  zslice_scatter.<locals>.inner_fnx  s   A:#/daic?"..S5;;7s)C5 0$7A:KKNN5<<#6D (?KKNN5<<#4ekkB 19KKNN'C5(8!TBEKK LLEKK0	 t$/**' #A

 yySM
 	
r\   rM  )r   rO  r  r   r+   r  normalize_start_endr^   r"   r  r7   r   rc   )r   r  r   r  r  r  src_sizerL  r  r  r  s   ` ````  @@@rZ   slice_scatterr  j  s    ;;=CMMO+++}}H
3
"Czz|C H11!S%EJE3AJJL!HS5[D1H5t<HSM
h
C"J,
 ,
\ ||~kkmAJJL!	 r\   c                j    t        | t        t        f      rt        |       dkD  rt	        | d         S | S r  )r`   r^   r   r   _unwrapr   s    rZ   r  r    s-    !dE]#A
qt}Hr\   r   rk   r   
pin_memoryc                   t        |d t        j                  fv d|        t        | d       t        t	               t
              rxs t        j                  nxs t        j                         g }t         t        j                        r fd}nt         t        t
        f      r fd}nt               dk(  s't         d   t        t
        f      rBt               dk  r4|j                  t        j                  t                             fd}n5t        j                  j!                  t        j"                   |            S t%        j&                  t)        |      ||	      S )
Nlayout=r  c                0    t        j                        S r   r  r(  rb   r   s    rZ   rL  ztensor.<locals>.inner_fn  s    >>$..r\   c                0    t        j                        S r   rH   r;  r  s    rZ   rL  ztensor.<locals>.inner_fn  s    <<e,,r\   r   r   c                      fdt              dk(  rt        j                  d      S  dt                    S )Nc           	     j   | |k  sJ || z
  dk(  rt        j                  |          S || z
  dz  | z   }t        j                  t        j                  t        j                  d   t
        j                        t        j                  |t
        j                               | |       ||            S )Nr(   r&   r   )rH   r;  r  r=  r9  rp   r:  )r  r  midbinary_searchrb   r   r(  s      rZ   r  z/tensor.<locals>.inner_fn.<locals>.binary_search  s    s{"{;!#<<UU;;U{q(50yyFFuQx=S%++6 "%-!#s+ r\   r   )r   rH   r;  )r(  r  rb   r   s   `@rZ   rL  ztensor.<locals>.inner_fn  s7     4yA~||Au-- CI..r\   r  rM  )r   rp   stridedr`   r  r   r:  get_default_dtyper   r   r#  r   rd   IntegerrI   rR   add_tensor_constantr6  r7   r   r>   )rb   r   rk   r   r  rK  rL  s   ``     rZ   r6  r6    s   v$..'&0BC:~|,'$-%$2002!F$$	/ 
D5#,	'	- 
Ta:d1gs|<TaemmCI./	/( ww**LLU6:
 	
 V$	 r\   c                z    t        | t              r|t        | |      } |t        | |      } | S t	        | ||      S )Nr  )r`   r:   r   r  r6  )rb   r   rk   s      rZ   	as_tensorr    sD    $	"D%(DT6*D$eF33r\   c                8    t        | t        j                        S rP  r5  )rb   s    rZ   long_tensorr    s    $ekk**r\   c                   ddl m}  |t        j                  j                  j
                  t        j                  j                  j                  d         }|J t        |      dk(  sJ |       t        t        |j                                     \  }}t        j                  |||       }t        j                  j                  |      |_        t        j                  j!                  |       t        j                  j                  j                  d   }t#        |t$        j&                  t$        j(                  t$        j*                  f      r|j,                  j.                  S t1        j2                  |      S )Nr   )resolve_unbacked_bindingsunbacked_bindingsr(   r  )r  r  rI   rR   r  r  rS   rT  r   r-  iterr   r+   DynamicScalarregister_bufferr   register_operationr`   rp   r  SymFloatSymBoolrX   r  r   r  )rb   r  r  binding_symkeypathbufferr  s          rZ   _local_scalar_denser    s   O 2	""AGG$8$8$=$=>Q$R ((( !Q&9(99&   %6%<%<%> ?@Kk7D9F''))&1FKGGv& ''


#
#E
*C#ennemmDExx}}}}S!!r\   c                     y r   r   )rb   r   s     rZ   _assert_scalarr  ,  s     r\   )rk   r   c                    y r   r   )r   r   r'  r   rk   r   s         rZ   _assert_tensor_metadatar!  5  s     r\   c                   | t        | t        t        f      st        d      rj                  t        t        t        f      rfd}nTt        t
        j                        rfd}n3t        j                               dk(  sJ j                         fd}t        j                  ||t        |            S )Nr   c                0    t        j                        S r   r  r(  r   r   s    rZ   rL  z_full.<locals>.inner_fnC  s    <<u--r\   c                0    t        j                        S r   r  r$  s    rZ   rL  z_full.<locals>.inner_fnH  s    >>%//r\   r   c                     g       S r   r   )r(  value_loaders    rZ   rL  z_full.<locals>.inner_fnO  s    ##r\   rM  )r`   r   r#  r   r   r   r   r   r   rO  r7   r   r^   )
fill_valuerk   r   r   rL  r   r'  s     `  @@rZ   _fullr)  <  s    Ej3,/GE74K%#u&	. 
E5;;	'	0 5>>#$)))((*	$ Dz	 r\   c                8     t        t        |            | fi |S r   create_tensor_liketensor_constructor)r   r(  r   s      rZ   r  r  Z  s     =0<=aJ6JJr\   c                "     d d d d dd d fd
}|S )NF)namesr   rk   r   r  r  c                   t        | d u d       t        |d t        j                  fv d|        t        | d       t        |      }|xs t        j                         }t        |      dk(  r6t        |d   t        t        t        j                  f      rt        |d         }|D ]  }t        |t        j                        sJ  |D cg c]  }t        j                  |       }}t        |||      S c c}w )Nnamed tensorsr   r  r(   r   )r   rp   r	  r>   r
  r   r`   r^   r   Sizer  r   r  r)  )	r/  r   rk   r   r  r  r   r  r(  s	           rZ   rZ  z!tensor_constructor.<locals>.innera  s     	5D=/26dEMM22gfX4FGz><0v&2002t9>ja4

2KLa>D  	3A!!U\\222	3)-.AQ..Z55 /s   C8r   )r(  rZ  s   ` rZ   r-  r-  _  s#     6. Lr\   )r/  r   r   rk   r  r  c                    t        | d u d       t        |      }t        |      dk(  r6t        |d   t        t
        t        j                  f      rt        |d         }t        |d ||||      S )Nr1  r(   r   r   r   rk   r  )	r   r>   r   r`   r^   r   rp   r2  empty_strided)r/  r   r   rk   r  r  r   s          rZ   r  r  {  sf     u}o.6"F
4yA~*T!WtUEJJ.GHT!W~d%v* r\   c                      dddddd fd
}|S )zZ
    Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
    NF)r   rk   r   r  r  c                  t        | d       t        |d t        j                  fv d|        || j                         }nt	        |      }|xs | j                         }t        | j                               } |||||      S )Nr  r   r  )r   rp   r	  r   r   rc   r^   r   )r   r   rk   r   r  r  r   creation_fns          rZ   _constant_likez*create_tensor_like.<locals>._constant_like  s     	z><06dEMM22gfX4FG=KKME 'E)1<<>AJJL!fV

 	
r\   r   )r8  r9  s   ` rZ   r,  r,    s     duTX
 r\   c                *    t        t        |             S r   r+  )r(  s    rZ   r  r    s    0<==r\   c                     d d d d d fd
}|S )Nr4  c               n   t        |t        t        f      sJ t        | d       t        |d t        j
                  fv d|        t        |      xs | j                         }|xs | j                         }|D cg c]  }t        j                  |       }}t        t        |      ||      S c c}w )Nr  r   )r`   r^   r   r   rp   r	  r   r   rc   r   r  r)  r>   )r   r   r   r   rk   r  r  r(  s          rZ   _new_constantz#new_constant.<locals>._new_constant  s     $u...z><06dEMM22gfX4FGU#4q{{})1<<>*./Qa //Zv!6tDD 0s   =B2r   )r(  r=  s   ` rZ   new_constantr>    s    tDT	E r\   r4  c          	     ~    || j                         }|| j                         }t        |d ||t        |      |      S Nr4  r   rc   r5  r>   )r   r   r   r   rk   r  s         rZ   	new_emptyrB    sF    }~V$ r\   c                  t        | t        t        f      sJ t        |t        t        t        d       f      sJ t	        | d       t	        |d t
        j                  fv d|        t        |      xs t        j                         }|xs t        j                  d      j                  }t        |      }t        d|||       }|j                          |j                  j                  }t        j                   |j                  dgt#        |       z        |_        t        |t$        j&                        sJ | D cg c]  }t)        j*                  |       } }|r#|D cg c]  }t)        j*                  |       c}nt$        j,                  j/                  |       }t%        j0                  ||| |      |_        |S c c}w c c}w )Nr  r   r8  r   )r(  rk   r   r   )rK  )rk   r   r   r'  )r`   r^   r   rP  r   rp   r	  r   r
  r6  rk   r>   r)  rd  rb   dataclassesreplacer   r+   r  r   r  rB  rC  r%  r   )	r   r'  r   r   rk   r  	pointwiser  r  s	            rZ   r5  r5    s{    dT5M***ftUDJ7888:~|,v$..'&0BC<5#:#:#<E/u||C(//F6"F6TJI^^  F%%fkk1#D	/JFKfb//000%)*ELLO*D*  #))Qa)11$7 
 NN	FM  +)s    G$Gc          	     ~    || j                         }|| j                         }t        ||||t        |      |      S r@  rA  )r   r   r'  r   r   rk   r  s          rZ   new_empty_stridedrH    sH     }~V$ r\   c                
   |D cg c]+  }t         j                  j                  j                  |      - }}t	        t        t        |            |j                        }t        j                  j                  | |      S c c}w )N)r  )rI   rR   r  r  sortedr   r   __getitem__r+   rl  rm  )r   r'  r  ro  s       rZ   copy_stridedrL    sa    5;<agg((+<F<%F,&2D2DEL??//<@@ =s   0B c                V    |j                  d      J d        t        |      | fi |S )Nr   z(dtype should be handled by decomposition)rU  r-  )r   r(  r   s      rZ   fullrN    s5    ::g*V,VV*)j)$9&99r\   c                   t        | t              sJ |j                         dk(  rt        | |j	                               S |j                         t        j                  k(  sJ | j	                         t              dk(  }t        | |      |rt        | dg      } dg| j                         |j                         fd}t        j                  | j                         | j                         ||j	                               S )Nr   r(   c                    t        |       } t        j                   |                }t        |       dk(  r|g} n|| <    |       S r  )r^   rH   indirect_indexingr   )r   
gather_idxr   index_loaderr   r  s     rZ   rx   zgather.<locals>.fn!  sL    3i**<+<d3iH
s8q=,C!CH}r\   rM  )r`   r:   	get_numelrB  r   r   rp   r:  r   r  r  rO  r7   r   rc   )	r   r   r(  sparse_gradr  rx   rS  r   r  s	    `    @@@rZ   gatherrV    s     a###AENN,--??+++::<DY!^F
3
'C1qcNs}}H$$&L ||~kkm~~	 r\   c                  	
 |rJ t        | t              sJ t        |t              sJ dt        |j                               v sJ | j	                         	|j	                         t        |j                               | j                         
g |j                         
dd  	
fd}t        j                  | j                         | j                         |      S )Nr   r(   c                    t        |       t              k(  sJ |  d         | d        }t        j                  |d         gg | d  z   } |      S )Nz != r   )r   rH   rQ  )r   	var_index
weight_idxindices_loaderindices_ndimr  weight_loaderweight_sizes      rZ   rx   zembedding.<locals>.fn?  s{    3x3x=(@SEhZ*@@("3}#56	++I{1~FG K
K
 

 Z((r\   rM  )
r`   r:   strr   rO  r   r   r7   r   rc   )weightr  padding_idxscale_grad_by_freqsparserx   r[  r\  r  r]  r^  s         @@@@@rZ   	embeddingrd  2  s    :fi(((gy)))C))+,,,,&&(M((*Nw'')*L//#K6!!#6k!"o6H) )   " 	 r\   c           
        t        d | D              s)J d| D cg c]  }||j                          c}        t        d | D              rt        d      t	        |       D cg c]  \  }}t        |t              s| }}}t        |      dkD  sJ d       d gt        |       z  }t        |t        |D cg c]  }| |   	 c}       D ](  \  }}|j                         |k7  rt        d      |||<   * ||fS c c}w c c}}w c c}w )Nc              3     K   | ]T  }|P|j                         t        j                  t        j                  t        j                  t        j
                  fv  V y wr   )r   rp   r:  r  r   r  r   rh   s     rZ   r   z.check_and_broadcast_indices.<locals>.<genexpr>P  sA      = 	
%++u{{EJJLLs   AAz)indices must be int64, byte or bool. Got c              3     K   | ]7  }||j                         t        j                  t        j                  fv  9 y wr   )r   rp   r   r  rg  s     rZ   r   z.check_and_broadcast_indices.<locals>.<genexpr>W  s1      78q}%**ekk22s   ?5?zFallback for bool indicesr   z"requires at least 1 non-None indexz.Fallback when indices is on a different device)r  r   r  r   r_   r`   r:   r   r   r   rc   )r  rk   rh   r   
valid_idxsnew_indicess         rZ   check_and_broadcast_indicesrk  O  s/      
 4G4eqWXWdQ[[]4e3fg   <C  ""=>> )' 2O1jI6N!OJOz?QD DD&3w<'KJ 1
3S1GAJ3S TU 1 <<>V#%&VWWA 
""# 5f P 4Ts   C=C='D D:Dc	           
     H   dt        dd        D ]  \  }	}
|
|	z
  dk7  sd t              D cg c]  \  }}|	| |    c}}g | t              t              z   d  d   }rz   nd | z   |d  z   f
d}|fS c c}}w )NFr(   Tr   c                  
 t        |       t              k(  sJ t              t        
      k(  sJ t              }g }d   }rdn|}d}t        d   dz         D ]}  }||k(  r||z  }|   *|t        |       k  sJ |j                  | |          |dz  }<|   }|J 
|   }|j                  t        j                   || |||z          |	              g || |d  }|S  |      S )Nr   r:  r(   r   wrap_neg)r   r   rd   rH   rQ  )r   r  	new_indexfirst_tensor_indexstart_offsetnext_idxrh   loaderr   r   indexed_sizer  indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizero  r  s            rZ   rx   z*index_output_size_and_inner_fn.<locals>.fn  sN   3x3{++++?#s<'8888;	+A.3q9K~b)A-. 	AL D qz!#c(***  X/A(+)))#A  ))s<,2EFG#!)		&

^
	 %,yE(92EEr\   )r   r_   r   )r  r  ry  rz  rv  ru  r  r   ro  previouscurrentrh   r  rq  rx   rw  rx  s    ````````      @@rZ   index_output_size_and_inner_fnr}  i  s    $ $ 1CD +'X"&*#+ ,5W+=MC6!9MKSKS&[)9C<O)O)Q"RSK'*!K/ ++,,-./ 	 F  FD ?_ Ns   
BBc                    t        | ||      \  }}}t        j                  | j                         | j	                         ||      S r  )index_impl_helperr7   r   rc   r   )r   r  r   rx  rL  r   s         rZ   
index_implr    sB    0GUCK1||~kkm	 r\   c                   t        |t        t        f      sJ | j                         t	        || j                               \  }}t        |      dkD  sJ d       |D cg c]  }||j                         nd  }}t        ||d      j                               }| j                         }t        t        |            D cg c]  }||   	||    }	}|rd|	v rd|vrt        d      t        t        |            D cg c]  }||   	 }	}t        ||||||	d ||	      \  }
fd}|
|fS c c}w c c}w c c}w )Nr   z Must have at least one valid idxz0index is out of bounds for dimension with size 0rn  c                       |             S r   r   )r   index_inner_fnr  s    rZ   rL  z#index_impl_helper.<locals>.inner_fn  s    s+,,r\   )r`   r^   r   rO  rk  rc   r   r   r   
IndexErrorr}  )r   r  r   ro  ry  rh   rv  rz  r  ru  rx  rL  r  r  s               @@rZ   r  r    sS   ge}---}}H9'1<<>RG^~"F$FF"KRSa!-q}}TASOS w~a01::<=KZZ\F',S\':U!gaj>TF1IULUl"q';KLL',S\':;!F1I;L;"@
#K- .009 T V <s   !D2
D7D7D<c                    	 t        | |d      S # t        $ r? | j                           t        t        j
                  j                  d      | |      cY S w xY w)NTr   Fr  )r  r   rd  r  r  r(  rP  r   r  s     rZ   r(  r(    sT    
!WD11 
			M

 1 1uMw
 	

s    AAAc                    t        | |d      S )NFr  )r  r  s     rZ   _unsafe_indexr    s    a..r\   c                6    t        t        |       |||dd      S )NTFr   may_realizeindex_put_impl_rw  r   r  r   
accumulates       rZ   	index_putr    s     a'6:Tu r\   c                6    t        t        |       |||dd      S )NFr  r  r  s       rZ   _unsafe_index_putr    s     a'6:U r\   c                    |j                         | j                         k7  rt        || j                               }|rt        | |      }t        | t	        |d   ||             S r  )rc   r  r   r  r  )r  r  r   r  s       rZ   index_put_as_masked_fillr  	  sV    T__..%!23D% T5UD9::r\   c                r   t        j                         }t        |      r\|s|rX|sdnd}t        j                  j
                  j                  j                  dd       x}r| d| }|t        j                  _        t        j                  t        j                  j
                  j                  | |||       | S )Nzindex put with accumulate.zdeterministic index put.stack_trace Found from : 
 )rp   $are_deterministic_algorithms_enabledr3   rI   rR   rS   rT  rU  disable_cudagraphs_reasonr+   IndexPutFallbackrV   )r  r  r   r  deterministicr   r  s          rZ   index_put_fallbackr    s    >>@MjM ! )+ 	
 ''..3377tLL;LE*;-8C,/),,33T7FJWKr\   c                $    t        | |||dd      S )NTr  r  r  r  r   r  s       rZ   
index_put_r  !  s    gvz4 r\   c                $    t        | |||dd      S )NFTr  r  r  s       rZ   _unsafe_index_put_r  (  s    gvzD r\   c           
     b   |rBd }d  ||       |j                         v r$t        fd|D              s|j                          |j                         dk(  rt	        |      dk(  r|d   j                         t        j                  t        j                  fv r_|d   }t        t	        |j                               t	        | j                                     D ]  }t        |d      } t        | |g||      S t        j                         rt        | |||      S |D ]E  }	|	|	j                         t        j                  t        j                  fv s7t        | |||      c S  | j                         }
t	        |
      }|rLt        | j                               r3|dk(  rt!        | dg      } t        | |||      } |dk(  rt!        | g       } | S t#        || j                               }	 t%        || j'                               \  }}|D cg c]  }||j+                         nd  }}t-        | t.              sJ | j                          |dk(  rt!        | dg      } t1        ||d      j                               }t        t	        |            D cg c]  }|
|   	 }}t3        |
|||||d |      \  }}t5        ||      }t7        j8                  | j'                         | j                         |j+                         |||rdnd 	      }t7        j:                  d t7        j<                  |       |
      }t>        j@                  jC                  |      |_"        t>        j@                  jG                  |       |dk(  rt!        | g       } | S # t(        $ r t        | |||      cY S w xY wc c}w c c}w )Nc                F   t        | t        j                        r| j                  } t        | t        j                        r| j                         } t        | t        j                        r| j                  } t        | t        j                        r| j                         S d S r   )	r`   r+   r:   rb   r!  r"  r  Bufferry  r   s    rZ   try_get_namez%index_put_impl_.<locals>.try_get_name2  sg    !R\\*FF!R[[)MMO!R]]+FF#-a#;1::<EEr\   c                   t        | t              rt        | j                  t        j                        r| j                  j                         } t        | t        j                        xr t        | j                  t        j                        xrh t        | j                  dd       xrO | j                  j                  j                  t        j                  j                  j                  j                  k(  S y)Nr  F)r`   r:   rb   r+   r!  r"  r  rl  r   r  rV   rp   rH   r  randpermr  )indices    rZ   indice_slice_from_randpermz3index_put_impl_.<locals>.indice_slice_from_randperm;  s     &),FKK1U002vr}}5 V"6;;@VY=V ++22eiinn6M6M6U6UU	 r\   c              3  .   K   | ]  } |        y wr   r   )r   r  r  s     rZ   r   z"index_put_impl_.<locals>.<genexpr>I  s      E
39&v.E
r  r(   r   r:  r  
atomic_addrk   r   rL  rK  output_indexerscatter_moder   r   rb   )$r  r  rd  rT  r   r   rp   r   r  r   r   r  r  r  r  rC   r  r   rk  rc   r   rO  r`   r:   r^   r}  r  r+   Scatterr  MutationLayoutSHOULDREMOVErI   rR   r  r   r  )r  r  r   r  r   r  r  rE  r   r(  r  x_ndimry  rh   rv  rz  ru  expected_vals_sizerL  scatterr  r  s                        @rZ   r  r  /  sz   	F	 !6!6!88 E
=DE
 B
 NN 	aLAAJ  "uzz5;;&??qzs4==?+S-AB 	'AT2&D	''tffjII 113!$DD  I!2uzz5;;6O!O%dGVZHHI ]]_F[FB4>>CSTQ;qc?D!$DQ;b>Dfdnn./FE"=T__&#
 LSSa!-q}}TASOSdI&&&LLN {D1# w~a01::<=K',S\':;!F1I;L;#A	$  F./F jj nn##%!%/\TG ,,T2F
 ''))&1FKGGv&{D"~Ka  E!$DDE T <s   )N
 
N'N,
N$#N$r  c                   	 t        | |dd      \  }}j                         | j                         		fd}t        j                  | j	                         | j                         ||      S )NFrn  c                     j                   t        j                  k7  r+t        j                          t        j                        }n        }t        j
                  | fd      S )Nc                                     S r   r   )_unsafe_index_fnr   self_loaders   rZ   r7  z8_unsafe_masked_index.<locals>.inner_fn.<locals>.<lambda>  s    K8H8M,N r\   )r   rp   r   rH   r   r?  )r   mask_valr  fillrE  mask_loaderr  s   ` rZ   rL  z&_unsafe_masked_index.<locals>.inner_fn  sK    ::#||K$4ejjAH"3'Hzz($NPTUUr\   rM  )r  rO  r7   r   rc   r   )
r  rE  r  r  rK  r   rL  r  r  r  s
    ` `   @@@rZ   _unsafe_masked_indexr    sw    "3gUU#FA ""$K""$KV V  nn	 r\   c           	         t        ||d      }| j                         }t        t        |            D cg c]#  }||   rt	        ||   ||    ||   dz
        nd % }}t        | ||d      S c c}w )Nr   r(   T)r  )r  r   r   r   r  r  )r   rE  r  r   masked_valuer  rh   clamped_indicess           rZ   #_unsafe_masked_index_put_accumulater    s    vq)LJJLE s7|$ 7>ajgaj58)U1X\2dJO  Q$OOs   (A-c                V    t        j                  |t        j                  ||             S r   )rH   rT  rU  r   minmaxs      rZ   r  r    s    ;;sCKKQ/00r\   c                P    t        |       }t        ||||      }t        ||       |S r   )rw  r,  copy_)r  r  r   r'  r(  r  output_views          rZ   as_strided_scatterr    s*    4[FVT6>BK	+sMr\   c                2    t        t        |       |||fi |S r   )scatter_rw  )r   r   r(  r  r   s        rZ   r  r    s    E!Hc5#888r\   r  include_selfc               :   t        |t              }t        | ||j                         t	        t
        j                  |r|j                         n
t        |            |r|j                         j                  nd|      rt        j                  | ||||||       |S y )Nznot implr  )r`   r:   rG   r   r	   rp   r   rP  rc   r+   ScatterFallback)r"  r  r   r(  r  r  r  src_is_tensors           rZ   scatter_fallbackr    s     sI.MU[[]#--/S	J!.J 	%	
 r\   r  c                  |dv sJ |[t        t        j                  t        j                  j
                  j                  j                        }t        || ||||      }||S |dk(  rd}n|dk(  rd}t        | ||||      S )N)Nr   multiplyr  r   sumr  prod)
r   r  r  rI   rR   rS   rV   _overloadnamer  scatter_reduce_)r  r   r(  r  r  r"  fallback_results          rZ   r  r    s    ....~dmmQWW-A-A-H-H-V-VW*sE3v
 &""	:	4eS&99r\   c                0    t        t        |       |||      S r   )scatter_add_rw  r   r   r(  r  s       rZ   scatter_addr    s    a#uc22r\   c                     t        | |||d      S )Nr  )r  r  s       rZ   r  r  $  s    1c5#u55r\   c                4    t        t        |       ||||fi |S r   )r  rw  )r   r   r(  r  reduction_typer   s         rZ   scatter_reducer  )  s    58S%nOOOr\   )r  c          
         |dv sJ t        t        j                  j                               dk(  r dt        j                  j                         v sJ d       t	        t
              rt               t        t        j                  j                   |||      }|r|S t	         t              sJ dt        |j                               v sJ t         j                               }|dk(  rt         dg       t	        t              r)t        j                               dk(  rt        dg      t	        |t              r)t        |j                               dk(  rt        |dg      }|j                         dk(  r S t                j!                          |j#                         t	        t              rj#                         nd  fd} fd	}	d
 }
|st%        j&                   j)                          j                          fd|j                         |d       }t%        j*                  d t%        j,                         |      }t.        j0                  j3                  |      |_        t.        j0                  j7                  |       t%        j&                   j)                          j                         |	|j                         | |
|            }t%        j*                  d t%        j,                         |      }t.        j0                  j3                  |      |_        t.        j0                  j7                  |       |dk(  rt         g         S )N)Nr  r  meanamaxaminr(   twozKaten.scatter_reduce_.two is not the unique overload of aten.scatter_reduce_r  r   r   c                    j                         }t        |      }t        |       }t        j                   |       |dk(  rdn|   d      |<   |S )Nr   r(   F)ro  )r   r   r^   rH   rQ  )r   r  r  indirect_idxr   rS  r  s       rZ   r  z'scatter_reduce_.<locals>.output_indexer\  sU    5zCy11DAIq5:
S r\   c                `    r |       S t        j                  j                               S r   rH   r;  r   )r   r  r  r  s    rZ   rx   zscatter_reduce_.<locals>.fnf  s*    c?" <<T^^%566r\   c                    | dk(  ry| J y )Nr  r  r   r  s    rZ   backend_reduce_strz+scatter_reduce_.<locals>.backend_reduce_strm  s    U? >!>r\   c                L    t        j                  dj                               S r  r  )r(  r  s    rZ   r7  z!scatter_reduce_.<locals>.<lambda>z  s    3<<4>>3C#D r\   r  r  )r   r  r  r   r`   r   r  r  r  r:   r_  r   r   r  rT  r  rd  rO  r+   r  rc   r  r  rI   rR   r  r   r  )r  r   r(  r  r  r  r  r  r  rx   r  zero_outr  r  rS  r  s   `` `          @@rZ   r  r  .  s   BBBBD  **,-2T))3355U UU	6 #vc"&  !O dI&&&C)****t}}DqyD1##y!c#,,.&9Q&>3n%#ENN,<(=(BUQC A
c
"CLLN$$&L&0i&@"dJ7 ::??$.."D>>#)
 ""006

 gg--f5	""6*
 jj nn~~%'/G ,,T2F
 ''))&1FKGGv&qyD"~Kr\   c                T   | j                          | j                         | j                          d  | j                         d   }D cg c]+  }t        j                  j
                  j                  |      - c}t        |      k(  sJ |}t        |      D cg c]
  \  }}||z   c}}t        |      D ]  \  }}	|		d|	z  |<    fdfd}
t        j                  | j                         | j                         |
g ||      S c c}w c c}}w )Nrp  c                   t        j                  | t        j                        } r8t        j                  | t        j
                  dt        j                              } t        j                  | t        j
                  |t        j                              } t        j                  | t        j                        } t        j                  | |d      S )N      ?Fr  )
rH   r9  rp   rR  r   r;  rN  r   r  rQ  )r   rW  r   exacts      rZ   scale_fnz$upsample_nearestnd.<locals>.scale_fn  s     NN1emm,3<<U]];<AGGAs||E5==9:LLEKK($$QE::r\   c                    |  d  }| d   } 
g |t        |      D cg c]  \  }}} 	|||       c}}}      S c c}}}w r   )r   )r   r   r  rh   r  r   i_sizes
inv_scalesnr  r  s         rZ   rx   zupsample_nearestnd.<locals>.fn  s`    H!HWaW#aW:UVVJAq$8Aq$'VW
 	
Vs   ArM  )realize_hintrO  r   rI   rR   r  r  r   r   r_   r7   r   rc   r   )r   rx  scales_xr  r  batchrh   o_sizesorW  rx   r  r  r  r  s      ``      @@@@rZ   upsample_nearestndr    s    NN}}HjjlA23GJJL1"EBIJQqww55a8JGx=AG$'$9:DAq!a%:Jh' (5%KJqM(	;
 
 ||~kkm!!!	 9 K
 ;s   0D'D$c                "    t        | ||fd      S )Nr(   r  r  r   rx  ra  s      rZ   upsample_nearest1dr    s    avi1==r\   c                $    t        | ||fdd      S )Nr(   Tr  r  r  r  s      rZ   _upsample_nearest_exact1dr    s    avi1DIIr\   c                $    t        | |||fd      S )Nr&   r  r  r   rx  scales_hscales_ws       rZ   upsample_nearest2dr    s     ax.BaHHr\   c                &    t        | |||fdd      S )Nr&   Tr  r  r	  s       rZ   _upsample_nearest_exact2dr    s     ax.BatTTr\   c                &    t        | ||||fd      S )Nr   r  r  r   rx  scales_dr
  r  s        rZ   upsample_nearest3dr    s     ax8.LPQRRr\   c                (    t        | ||||fdd      S )Nr   Tr  r  r  s        rZ   _upsample_nearest_exact3dr    s#     	;8X6!4 r\   c                ,     t         fd|D              S )Nc              3  J   K   | ]  }t        j                  |        y wr   r  )r   r   r   s     rZ   r   z$_create_constants.<locals>.<genexpr>  s     6Aa'6s    #)r   )r   ri   s   ` rZ   rQ  rQ    s    6666r\   c                    | j                         | j                         fd}t        j                  | j	                         | j                         |      S )Nc                    t        |       } t        |       t              k(  sJ D ]  }|   dz
  | |   z
  | |<     |       S r  )r^   r   )r   r   r  r  r  s     rZ   rt  zrev.<locals>.loader  sW    3i3x3u:%%% 	3Cc
Q#c(2CH	3 }r\   rM  )rO  r   r7   r   rc   r   )r   r  rt  r  r  s    ` @@rZ   revr    sM     }}HJJLE ||~kkm	 r\   c                   d } |       ryt        |      dk7  st        | j                               dk7  ry| j                          t        | t        j
                        rt        | j                  t        j                        rt        | j                  j                  t        j                        s>t        j                  rNt        | j                  j                  t        j                        r | j                  j                  j                  sy| j                          t	        j                  |       \  }}|j                  }|d   dk7  ry|d   dk7  s|d   dk7  s|d   dk7  ry|d   }|dk(  ry|d   }|j                   d   }	||	|z   k  ry| j                  j                  j                  }
|j                   d   |j                   d   |z   g}|t"        j$                  j&                  |
<   t)        | ||j                  |j*                        }t-        |d|	|	|z         }t/        ||       t0        d	   d
xx   dz  cc<   |S )z
    This optimization changes the semantics of padding from 'clone'
    style to 'view' style.

    Thanks to functionalization, this change can still maintain numerical
    correctness.
    c                    t         j                  j                  } | yt        | j                        }t        |      dk(  rD|d   j                  t        j                  j                  t        j                  j                  fv ryy)a  
        Conservatively check if padding can be fused with downstream op.
        1. if the downstream op is a sum, then there is little benefit to
           do inplace padding
        2. if the downstream op is a matmul, doing inplace padding can
           save membw.
        Tr(   r   F)rI   rR   rS   r   rT   r   rV   r  mmr  addmm)rS   rT   s     rZ   _padding_can_be_fusedz6inplace_constant_pad_nd.<locals>._padding_can_be_fused#  sk     ww++l(()u:?uQxGGOOJJ3
  
 r\   Nr   r&   r(   r   r   )r   r  r  inductorinplace_padding)r   r   rd  r`   r+   r:   rb   r  r  r)   can_inplace_pad_graph_inputInputBufferr   freeze_layoutr$  r'  r   rI   rR   buffer_to_padded_sizer,  r  r  fill_r   )r   paddingr(  r  r   r   r  npadstride0rowsizebufnamepadded_size	resized_xsliced_xs                 rZ   inplace_constant_pad_ndr.    s   (  7|qC

-2
 IIK
 q",,'!&&"--0qvv{{B$5$5622qvv{{BNN; vv{{OO((+IAvmmGqzQqzQ'!*/WQZ1_1:DqyajGkk!nG4ffkkG;;q>6;;q>D#89K-8AGG!!'*		I iQg7T>JH	(JZ*+q0+r\   c                   t        |      dz  dk(  sJ t        d |D              rt        |       S t        j                  rt        | |      }|r|S | j                         }t        t        t        t        |d d d   |dd d                           t        |      t              z
  g D ]?  \  }}j                  t        j                  j                  j                  |      |f       A t        |d        }g t        |d        D ]C  \  \  }}	}
j                  |
       |j                  t        j                   |
|z   |	z                E t        |      t        |      k(  sJ  t#        | j%                                     fdfd}| j'                         t)        j*                  | j-                         | j%                         ||      S )Nr&   r   c              3  &   K   | ]	  }|d k(    ywr   Nr   r   rK  s     rZ   r   z"constant_pad_nd.<locals>.<genexpr>z  s     
#a16
#r  r(   c                B    g }t         	d        D ]J  \  }\  }}}|dk7  r|j                  t        |d             |dk7  s0|j                  t        ||             L t	        j
                  t        j                  |      }t        j                  | 
fd      S )Nr   c                             S r   r   )r(  r  s   rZ   r7  z/constant_pad_nd.<locals>.mask.<locals>.<lambda>  s     r\   )	r   rd   range_mask_lowrange_mask_highr   r  rH   r>  r?  )r(  rE  r   r  r  r  boundsr(  
mask_sizesr  r  s   `     rZ   rE  zconstant_pad_nd.<locals>.mask  s    (+E!"Ivz(J 	:$C#tfaxN323qyOC89		:
 $/zz$ 7DDr\   c                    t        | d        }t        | d        D ]  \  }\  }}|j                  ||z
          t        |      t        |       k(  sJ  |      S r   )r^   r   rd   r   )r(  rp  r   r  _highbounds_precomprE  r  s        rZ   	offset_fnz"constant_pad_nd.<locals>.offset_fn  si    rO	!$U12Y!? 	(C#uS3Y'	(9~U+++Ir\   rM  )r   r  rw  r)   r   r.  r   r^   r  r   rd   rI   rR   r  lookup_precomputed_sizer   r  r   r   rO  r7   r   rc   )r   r&  r(  rf   r  lhrx  r  r  r   r<  r7  r;  rE  r8  r  r  s     `         @@@@@@rZ   r  r  w  s   L1"""

#7
##Qx%a*=J JJLE(4GCaCL'!$Q$- @ABCFE
S[ A 68N P1qww//GGJANOP uRay/KJ qr3 <dT$5<<s
T(9:;< {s5z)))-q{{}-j9JE E }}H||~kkm	 r\   c                    t        j                  t        j                  | t        j                        t        j                  t        j                  |      t        j                              S r   )rH   r<  r9  rp   r:  r   r  )rh   r  s     rZ   r5  r5    s@    66q%++&u}}S)5;;7 r\   c                    t        j                  t        j                  | t        j                        t        j                  |t        j                              S r   )rH   r=  r9  rp   r:  )rh   r  s     rZ   r6  r6    s7    66q%++&tU[[) r\   c                V    t        j                  t        | |      t        | |            S r   )rH   r>  r5  r6  )rh   r  r  s      rZ   
range_maskrC    s'    88q#4  r\   c                      j                          d   j                         xs dgz   fd}|S )Nr   c                N   | d   |  d  t        j                  t        j                  t	              D cg c]  }t        |   |   
|   z   
|          ! c}      }	rt        j                  |fd      S t        j                  |fd      S c c}w )Nc                 4     t               g       S )Nr   )constant_boundary_condition)r   ihpad_fill_valueprefixr   s   rZ   r7  z;constant_boundary_condition.<locals>.load.<locals>.<lambda>  s$    O3A~3O"fNrN r\   c                      g        S r   r   )rI  rK  r  s   rZ   r7  z;constant_boundary_condition.<locals>.load.<locals>.<lambda>  s    (>V>b>*B r\   )r   r  rH   r>  r   rC  r?  )r(  rh   rE  rI  rK  r   r(  r?  rJ  r&  	padding_hr   r  s      @@rZ   r@  z)constant_boundary_condition.<locals>.load  s    uC45\HHLQRUJWqZ1qtil2Yq\MBW
  JJ 
	
 D"BJO
	
 Xs   $B"
)r   rO  )	r   r(  r&  rJ  r   r@  r?  rM  r  s	   ````` @@@rZ   rH  rH    sE     	


cTUA}}H$A39I
 
( Kr\   c                x   t        | d||   z  z   ||   dz
  z
  ||   dz
  z   ||         }|r
t        | d||   z  z   ||   dz
  z
  d||   dz
  z  z   ||         }t        j                  j                  j	                  |dz
  ||   z  | z
  ||   z
        dk\  r>|dz  }t        j                  j                  j                  d|||   z  | z
  ||   z
         t        j                  j                  j	                  ||z
        dk(  r0t        j                  j                  j                  ||       d}||fS |}||fS )Nr&   r(   r   F)r"   rI   rR   r  r  r  r  )r   rh   kernel_sizer'  r&  	ceil_modex_outx_alts           rZ   pooling_sizerS    sW   	A
Nk!nq01VAY]CVAYE GAJ+a.1"45VAY]8KKVTUY
 77%%uqyF1I&=&AGAJ&NOSTTQJEGG&&q%&)*;a*?'!**LM77%%eem49GG))%7I ) E)r\   c                b    t        | d      } | d   | d   z  }|dkD  xs t        d |D              S )Nr&   r   r(      c              3  &   K   | ]	  }|d kD    ywr  r   r  s     rZ   r   z:should_fallback_max_pool2d_with_indices.<locals>.<genexpr>  s     $=qQU$=r  )rD   r  )rO  dilationwindow_sizes      rZ   'should_fallback_max_pool2d_with_indicesrY    s;    {A.Ka.;q>1K"=$=H$=!==r\   assert_fallbackc                  |dk(  rddg}|dk(  rddg}|s|}t        |d      }t        |d      }t        |d      }t        |d      }t        | t              sJ t        |      dk(  sJ t        |      dk(  sJ t        |      dk(  sJ t        |      dk(  sJ t        | j	                               dv sJ t        ||      }|||k(  sJ |||||fS )Nr   r(   r&   r  )rD   r`   r:   r   r   rY  )r   rO  r'  r&  rW  r[  use_fallbacks          rZ   max_pool2d_checksr^    s     !|a&1}q6{A.K&!$F7A&GHa(Ha###{q   v;!w<1x=Aqzz|&&&:;QL"...<??r\   c           
     x   | j                          | j                         ^ }}}t        |d||      \  }	}
t        |d||      \  }}| j                  }|t        j
                  u rdn5|j                  rt        d      nt	        j                  |      j                  }t        |      |	|gz   }d   s	d   s|
s|rt        | |d      n| j                         dfd}t        j                  d| | j                         |||||	      }t        j                  d
| | j                         t        j                   ||||	      }t#        |j$                  j$                  t              r|j'                          t#        |j$                  j$                  t              r|j'                          ||fS )Nr   r(   F-infr&   rG  c                    | d   }|  d  }t              D cg c]  }||   |   z  ||   z   |   z
   }} 	g ||      S c c}w r   )r   )
r   reduction_idxrK  bhrh   rI  r   r&  r'  r  s
         rZ   fn_innerz*_max_pool2d_with_offsets.<locals>.fn_inner8  sm    Usd#ZINsTAbefQi-"22WQZ?TT&2'' Us   Ar  )r  
input_noderk   	dst_dtypert  rL  rK  reduction_rangesargmax)r  r   rS  r   rp   r   r  r#  r  r  r^   rH  rO  r8   r   rc   r:  r`   rb   rd  )r   rO  r'  r&  rW  rP  r  r?  wh_out
ceil_mode1w_out
ceil_mode2r   	min_valuer  rd  r0  offsetsr   r  s     ``               @@rZ   _max_pool2d_with_offsetsrp    s    NN::<LUAq$Q;SE:$Q;SE:GGE EJJ 	$66eFmEKK<N<R<R  E{eU^+HqzWQZ:.q)C==?
C( ||~$	F ||~++$	G &++""I.',,##Y/7?r\   c           	         t        | ||||d      \  }}}}}t        j                  d      5  t        | |||||      \  }}|t	        |t
        j                        fcd d d        S # 1 sw Y   y xY w)NFrZ  rU  )unroll_reductions_threshold)r^  r)   r   rp  r   rp   r  )	r   rO  r'  r&  rW  rP  r   r0  ro  s	            rZ   #_low_memory_max_pool2d_with_offsetsrs  \  s     1B	1-K(A 
"	5 	52
 x44	5 	5 	5s   /A''A0c                    | j                         fdfd}t        j                  | j                         t        j
                  || j                               }|S )Nc                0   t        j                  	t        j                        }t        j                  |d   z  
d   z
  t        j                        }t        j                  |d   z  
d   z
  t        j                        }|| z   }||z   }||z  |z   S r  )rH   r9  rp   r:  )h_incw_incrc  bww_inhbasewbaserI  iwinput_widthr&  r'  s            rZ   increments_to_indexzF_low_memory_max_pool2d_offsets_to_indices.<locals>.increments_to_index  s    ~~k5;;7rF1I~
:EKKHrF1I~
:EKKHU]U]Dy2~r\   c                    | ^ }}} 
g |||      }t        j                  	t        j                        }||z  }|||z  z
  } ||||      S r   rH   r;  rp   r  )r   rK  rc  rx  r  kw_constrv  rw  r~  kernel_widthoffsets_loaders           rZ   offsets_to_indiceszE_low_memory_max_pool2d_offsets_to_indices.<locals>.offsets_to_indices  sf    R 1& 1" 1b 12<<ekk:("%(*+"5%R88r\   rM  )rO  r7   r   rc   rp   r:  r   )	ro  r  r}  r'  r&  r  r  r~  r  s	    ````  @@rZ   )_low_memory_max_pool2d_offsets_to_indicesr  {  sX     ((*N9 !!#kk#!	G Nr\   c                    t        | ||||      \  }}}}}t        d |D              rt        | |||||      S t        | |||||      \  }}t	        ||d   | j
                  d   ||      }	||	fS )Nc              3  &   K   | ]	  }|d kD    ywr  r   r  s     rZ   r   z*max_pool2d_with_indices.<locals>.<genexpr>  s     
#Q1q5
#r  )rP  r:  )r^  r   fallback_max_pool2d_with_indicesrp  r  r  )
r   rO  r'  r&  rW  rP  r   rf   ro  r  s
             rZ   max_pool2d_with_indicesr    s     1B	;1-K(A 
#(
##/{FGX
 	
 ,	;9LC 8R!''"+vwG <r\   c                   dk(  rddg|dk(  rddg}st        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |      dk(  sJ t        |j                               dv sJ | j	                          | j                         }t        |t              rt        |j                  j                  t              r|j                  j                  }	t        j                  d t        j                  |	j                         |	j                         |	j                               |	      }
|
j                          |
j                         }n|j                         }|d uxr |d   dk(  xs |d uxr |d   dk(  }t        d |D              rt!        | ||||      S |j                         ^ }}| j                         ^ }|j#                         | j#                         t%        |j                               }t'        fdt)        d   dz        D              t'        fd	t)        d   dz        D              z  }|d
kD  rt!        | ||||      S |j                         fd}t        j*                  | j                         | j                         ||      }|rt        j,                  j/                  |      S |S )Nr   r(   r&   r  )rk   r   r   r  c              3  &   K   | ]	  }|d k7    ywr  r   r  s     rZ   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>  s     
$a16
$r  c           
   3  t   K   | ]/  }t        |d    z  t        d |d    z
  d    z        z
  d       1 ywr   r(   Nr  r   r?  rO  r'  s     rZ   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>  F       	ANSQQ%7F1I$EFFJ   58c           
   3  t   K   | ]/  }t        |d    z  t        d|d    z
  d    z        z
  d        1 ywr(   r   Nr  r   ri  rO  r'  s     rZ   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>  r  r  rU  c                   | ^ }}}t        j                  |z  |z   t        j                        }|d   z   }|d   z   }t        j                  t	        |d   z
  d   z   d         t        j                        }t        j                  t	        |d   z
  d   z   d         t        j                        }t        j                  t	        |d         dz   t        j                        }t        j                  t	        |d         dz   t        j                        }t        j
                  |t        j                  dt        j                              }t        j
                  |t        j                  dt        j                              }t        j                  |t        j                  t        j                              }t        j                  |t        j                  t        j                              }d }	t              D ]3  }
t              D ]!  }t        j                  |t        j                  |
t        j                              }t        j                  |t        j                  |t        j                              }g |t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d   d      t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d   d      } |      } |      }t        j                  ||      }|	;t        j                  ||t        j                  dt        j                              }	t        j                  t        j                  t        j                   ||      t        j                   ||            |      }t        j                  |t        j                  |	|      |	      }	$ 6 |	J |	S )Nr   r(   Fr  r:  r8  )rH   r9  rp   r  r"   rT  r;  rU  r   r   rQ  rk  r  r  rR  r>  r=  )r   rK  r?  ri  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   rE  grad_loaderh_window_sizer[  indices_sizerO  r&  pooled_heightpooled_widthr'  w_window_sizewidths                      rZ   rx   z,max_pool2d_with_indices_backward.<locals>.fn  sA   A^^AIM5;;?

N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM' #	WC]+ "WWWWcll3&DEWWWcll3&DE))Bs||Au{{7S(TU$R(# ))Bs||Au{{7S(TU$R(#
  .j9'
3	|Z8#"yyy#,,sEMM*J H 88FF2u-FF2u- D  #yyswwx/KXVHE"W#	WH ###r\   rM  )r`   r:   r   r   r  r  rb   r7   r+   r  rB  rc   r   decide_layoutr  r  )fallback_max_pool2d_with_indices_backwardrO  r^   r  r   r   rl  r  )grad_outputr   rO  r'  r&  rW  rP  r  	gO_striderb   x_bufferx_strideis_channels_last_batch_heightr   r  rX  rx   rf   r  r  r[  r  r  r  r  r  s     ```               @@@@@@@@rZ    max_pool2d_with_indices_backwardr    s    !|a&1}q6a###{q   v;!w<1x=Aqzz|&&& ,,.I!YJqvv{{I$Fvv{{$$$$(nn&]]_
 
 	 &&(%%' ,A!1A 3)A,!"3  
$8
$$8K(Iw
 	
  jjlVWe&1&:&:&<#Q|((*N))+KAJJL!H {1~)* M  {1~)* M
  -/KR8K(Iw
 	
 ##%L9 9v 

%%'##%	C 44S99
r\   c                6    | j                         fd}|S )Nc           
        	
 |\  	|\  
|\  }}t        j                  t        j                  t        j                  z   t        j
                        t        j                  |t        j
                              t        j                  t        j                  
	z   t        j
                        t        j                  |t        j
                                    }t        j                  |	 
fd      S )Nc                 ,     g  z   z         S r   r   )h_start_indexrI  r|  rK  w_start_indexr  s   rZ   r7  z3pad_adaptive_loader.<locals>.load.<locals>.<lambda>m  s&    HNvN}r'9N=2;MNO r\   )rH   r>  r=  r9  rp   r:  r?  )rK  
incrementsstart_indicesend_indicesh_end_indexw_end_indexrE  r  rI  r|  r  pad_valr  s   `      @@@@rZ   r@  z!pad_adaptive_loader.<locals>.load[  s    B'4$}#. [xxFF}r15;;?{EKK8 FF}r15;;?{EKK8	
 zzOO
 	
r\   rO  )r   r  r@  r  s    ` @rZ   pad_adaptive_loaderr  X  s    }}H
, Kr\   c                    t        j                  | ||      }t        j                  |||      }t        j                  | ||      }t        j                  |||      }	||||	fS )N)out_diminp_dim)r   r  )
start_index	end_indexh_inry  rj  rl  r  r  r  r  s
             rZ    compute_indices_adaptive_poolingr  t  sa    %%k5$OM##IudKK%%k5$OM##IudKK+}kAAr\   c                d    |\  }}|\  }}	t        | |||||	      \  fd}
|
S )Nc                   | ^ }}} |      } |      } |      } |      }d }	t        j                  t        d         t        d               D ]$  \  }
} |||
|g||g||g      }|	|}	 ||	      }	& |	S r  )r   productr   )r   rt  rK  rc  rx  r  r  r  r  r0  rI  r|  r  h_end_index_fnh_start_index_fnkernel_maxes
pooling_fnw_end_index_fnw_start_index_fns                rZ   rx   z _adaptive_pooling_fn.<locals>.fn  s    R(,$R((,$R(''l1o(>lSTo@VW 
	1FBR.k*	C ~#C0
	1 r\   r  )r  r  r  in_sizes	out_sizesr  r  ry  rj  rl  rx   r  r  r  r  s     `  `     @@@@rZ   _adaptive_pooling_fnr  ~  sP     JD$LE5 	)YdE5	
 . Ir\   c                h   
 |\  }|\  }}t        | ||||      \  

fd}	|	S )Nc                   | ^ }}} |      } |      } |      } |      }d }	d }
t        j                  t        d         t        d               D ]  \  }} ||||g||g||g      }t        j                  ||z   z  |z   |z   t
        j                        }|
|}
n+t        j                  t        j                  ||	      ||
      }
|	|}	| ||	      }	 |
S r  )	r   r  r   rH   r9  rp   r:  r  gt)r   rt  rK  rc  rx  r  r  r  r  maxvalmaxindexrI  r|  r  r(  r  r  r  r  r  ry  r  s                  rZ   rx   z)_adaptive_pooling_fn_with_idx.<locals>.fn  s   R(,$R((,$R(''l1o(>lSTo@VW 	1FBR.k*	C NN#t+m;b@%++E  99SVVC%8%J~#C0)	1, r\   r  )r  r  r  r  r  r  r  rj  rl  rx   r  r  r  ry  r  s     `  `    @@@@@rZ   _adaptive_pooling_fn_with_idxr    sQ     JD$LE5 	)YdE5	
! !F Ir\   c                     j                         t        j                  k(  rt        d      t	         t
              sJ t        |      dk(  sJ  j                           j                         ^ }}}t        j                  j                  j                  |      }t        j                  j                  j                  |      }|\  }}||k(  r||k(  rt               S |dk(  s|dk(  r2g |||}t        | j                          j                               S ||z  dk(  r||z  dk(  r||z  ||z  g}t!         |      S t#        ||z   dz
  |      }	t#        ||z   dz
  |      }
t%        |      ||gz   } j                         }|	|
z  }|dkD  rt'         |      S d }d }t)        |||	|
g||g||gt*        j,                  	      t/        t1                      fd
}t3        j4                   j                         |||      }|S )Nz0'adaptive_avg_pool2d' not implemented for 'Long'r&   r   r  r(   rU  c                     t        | |z  |      S r   r"   r(  r  r  s      rZ   r  z)_adaptive_avg_pool2d.<locals>.start_index      733r\   c                2    t        | dz   |z  |z   dz
  |      S r  r  r  s      rZ   r  z'_adaptive_avg_pool2d.<locals>.end_index
  "    g-7!;WEEr\   r  r  r  r  r  r  c                ^    t        j                   | t                     |             S r   )rH   truedivr  )r   fn_sumones_loaderr   s    rZ   rx   z _adaptive_avg_pool2d.<locals>.fn  s-    {{3+A./[1I
 	
r\   rM  )r   rp   r:  r0  r`   r:   r   r  r   rI   rR   r  r  rw  r  rc   
avg_pool2dr=   r^   fallback_adaptive_avg_pool2dr  rH   r   r  	ones_liker7   r   )r   rx  r  r  ry  rj  rl  o_sizerO  h_kernel_maxw_kernel_maxr  r   rX  r  r  rx   rvr  r  s   `                 @@rZ   _adaptive_avg_pool2dr    s   {{}#MNNa###{q   NNUD$7711$7D7711$7DLE5 u}QxzUaZ'5'%''V1;;=HHe|qTE\Q.u}dem4![))D5L1,u5LD5L1,u5LE{eU^+HKKME-KR+A{;;4F ""L1%.77F &il3K

 
		||~	
B Ir\   c                     j                         t        j                  k(  rt        d      t	         t
              sJ t        |      dk(  sJ  j                           j                         ^ }}}t        j                  j                  j                  |      }t        j                  j                  j                  |      }|\  }}|dk(  s|dk(  r\g |||}t        | j                          j                               t        |t        j                   j                               fS ||z  dk(  r||z  dk(  rt        t!        ||z   dz
  |      }t!        ||z   dz
  |      }	t#        |      ||gz   }
 j                         }||	z  }|dkD  rt%         |      S d }d }t'        ||||	g||g||gt(        j*                  	      t-        ||||	g||g||gt(        j*                  	       fd
} fd}t/        j0                   j                         |||
      }t/        j0                   j                         t        j                  ||
      }||fS )Nz,adaptive_max_pool2d not implemented for Longr&   r   r  r(   rU  c                     t        | |z  |      S r   r  r  s      rZ   r  z(adaptive_max_pool2d.<locals>.start_indexQ  r  r\   c                2    t        | dz   |z  |z   dz
  |      S r  r  r  s      rZ   r  z&adaptive_max_pool2d.<locals>.end_indexT  r  r\   r  c           	     <     | t        t        d                  S Nr`  r  r#  )r   inner_func_max_valr   s    rZ   inner_fn_max_valz-adaptive_max_pool2d.<locals>.inner_fn_max_vali      !#':1eFm'LMMr\   c           	     <     | t        t        d                  S r  r  )r   inner_func_max_idxr   s    rZ   inner_fn_max_idxz-adaptive_max_pool2d.<locals>.inner_fn_max_idxl  r  r\   rM  )r   rp   r:  r0  r`   r:   r   r  r   rI   rR   r  r  r  rc   
ValueErrorr=   r^   fallback_adaptive_max_pool2dr  rH   rT  r  r7   r   )r   rx  r  r  ry  rj  rl  r  r  r  r  r   rX  r  r  r  r  r  rir  r  s   `                  @@rZ   adaptive_max_pool2dr  ,  sL   {{}#IJJa###{q   NNUD$7711$7D7711$7DLE5zUaZ'5'%''V1;;=H%%++allnK
 
 	
 e|qTE\Q.D5L1,u5LD5L1,u5LE{eU^+HKKME-KR+A{;;4F ."L1%.;; 7"L1%.;;NN 
		||~!	
B 
		||~kk!	
B r6Mr\   c                              t        z
  dz
         j                          fd}|S )Nr(   c                V    g | dz
  z
        }t        j                  |j                               }t        j                  j                               }t        j                  ||z   |z        t        j                  ||z        z
  }t        j                  |t
        j                        }t        j                  |t        j                  dz
  t
        j                              }t        j                  ||t        j                  	
z
  t
        j                              S r  )	rH   r9  r   r  r   rp   r:  r=  r  )rK  rh   samplei_expr
alpha_exprseq_irE  r5  r   in_sz	kernel_szndimsout_szsamplessamples_loaders          rZ   r@  z)_fractional_pooling_offsets.<locals>.load  s     :& :%!)c/ :;7#4#4#67^^E7+<+<+>?
		6F?j89CIIZ=
 
 UEKK0vvNN6A:u{{3
 yyucnnUY5F&TUUr\   )r$   rO  )	r
  r  r	  r  r   r  r@  r5  r  s	   `````` @@rZ   _fractional_pooling_offsetsr    sT    C[F#JE#Iuy(&1*5E((*NV V Kr\   c                   | j                          | j                         ^ }\  }}|\  }}||z  dk\  rt        | ||      S t        j                  t
        |g|d      }	 |	d       |	d      | j                         fd}
t        |      ||gz   }t        j                  | j                         | j                         t        j                  |
d	      |
      }t        j                  | j                         t        j                  t        j                  |
d	      |
      }||fS )NrU  r&   )r
  r  r	  r  r  r   rG  r(   c           	     t   | ^ }}}t        j                   ||            }t        j                   ||            }d }d }t        j                  t	        d         t	        d               D ]  \  }	}
 g |||	z   ||
z         }|rt        j
                  ||	z   z  |z   |
z   t        j                        }||}nRt        j                  t        j                  t        j                  ||      t        j                  |            ||      }||}t        j                  ||      } |r|S |S r  )rH   rQ  r   r  r   r9  rp   r:  r  or_r  r  rT  )r   return_indexrK  rc  rx  r  r  r  r  rI  r|  r  r(  
h_index_fninp_hinp_wrO  
w_index_fnr  s                rZ   rx   z!fractional_max_pool2d.<locals>.fn  s:   R--j.DeL--j.DeL''k!n(=u[QR^?TU 	2FBLVL]R%7L9KLMC"R'50=@2Eu{{ #$H"yysF 3SYYs^DeX H ~S&1	2  OMr\   F)r  rM  T)r  r   fallback_fractional_max_pool2dr   r  r  rO  r^   r7   r   rc   r   rp   r:  )r   rO  rx  random_samplesr  kernel_hkernel_wrj  rl  gen_offsets_for_dimrx   r  r  r  r  r  r  r  r  s    `            @@@@@rZ   fractional_max_pool2dr    s2   NN::<UE5$HhLE5(b -{K
 	
 $++#en %+J$+J}}H : E{eU^+H			||~kkm""2E:	
B 
		||~kk""2D9	
B r6Mr\   c                b     j                           j                         ^ }}}t        j                  j                  j                  |      }t        j                  j                  j                  |      }|^ }}}	||z  dk(  r||	z  dk(  rt         ||z  ||	z  gd      S t        ||      }
t        ||	      }d fd}t        ||
|g||g||	gt        j                         fd}t        j                   j                          j                         |t        |            }|S )	Nr   r(   )divisor_overridec                F    t        | |z  t        j                  |            S r   )r!   r   r  r  s      rZ   r  z0upsample_nearest2d_backward.<locals>.start_index  s    uwg(>??r\   c                     | dz   ||      S r  r   )r(  r  r  r  s      rZ   r  z.upsample_nearest2d_backward.<locals>.end_index  s    EAI99r\   r  c                (     | t                    S r   )r  )r   r  r   s    rZ   rx   z'upsample_nearest2d_backward.<locals>.fn  s    c.q122r\   rM  )r  r   rI   rR   r  r  r  r=   r  rH   r   r7   r   rc   r   r^   )r   rx  
input_sizer
  r  r  r  r  out_hout_wr  r  r  rx   r  r  r  s   `              @@rZ   upsample_nearest2d_backwardr#    s"    NNJJLVUEGG2259EGG2259E&VUEu}eemq0!euneun=PQRR5%(L5%(L@: ""L1%.77F3 
		||~kkmJ	
B Ir\   c           
     (    t        | ||||||d      S )Nr&   rG  _avg_poolndr   rO  r'  r&  rP  count_include_padr  s          rZ   r  r    )     		 	r\   c           
     (    t        | ||||||d      S )Nr   rG  r%  r'  s          rZ   
avg_pool3dr+  /  r)  r\   c                v   ssdgz  t              t              t              t        | t              sJ t              k(  sJ t              k(  sJ t              k(  sJ t        | j	                               dz   dz   fv sJ | j                          | j	                         d   }| j	                          d  t        t              D 	cg c]  }	t        |	   |	|       c}	 \  }
}t              st        |      rt        | d      d}n| j                         d}t        |      t        |
      z   }| j                         t        j                  t         j"                        }|dkD  r4dk(  rt$        }nd	k(  rt&        }nt)        d
        || ||      S fd|r|r)|r|n|j*                  rdz  fd}nfd}nf	d}t-        j.                  | j1                         ||      }|S c c}	w )Nr   r(   r&   r8  rG  TFrU  r   zUnknown dim: c           	     T   | d 	  }| 	 d  }d }t        j                  t        	      D cg c]  }t        
|          c} D ]V  }t        	      D cg c]  }||   |   z  ||   z   |   z
   }} |g ||      }||}At        j                  ||      }X |S c c}w c c}w r   )r   r  r   rH   r   )r   rt  rK  r  totalrh   rI  r   r  r   rO  r&  r'  s            rZ   r  z_avg_poolnd.<locals>.fn_sum  s    UsdJ##U3Z%PeKN&;%PQ 	,BBG*MQ1Q4&)#be+gaj8MCM6C)C}U+	,  &QMs   B B%c                f    t        j                   |       t        j                              S r   )rH   rN  r;  )r   r   r  rW  r  s    rZ   rx   z_avg_poolnd.<locals>.fn  s&    wwvc84cll5%6PQQr\   c                f    t        j                   |       t        j                              S r   )rH   truncdivr;  )r   divisorr   r  r  s    rZ   rx   z_avg_poolnd.<locals>.fn  s'    ||F3$93<<QV;WXXr\   c                F  	 | 	 d  }g }t        	      D ]  }||   |   z  |   z
  }t        j                  ||   z   |   |   z         }s/t        j                  |d      }t        j                  ||         }t	        j
                  ||z
  t        j                        }|j                  |        t        j                  t        j                  |      }
j                  rt	        j                   |       |      S t	        j                   |       |      S r  )r   r   MinMaxrH   r9  rp   r  rd   r   r  rN  r  r  r1  )r   rc  divide_factorsrh   hstarthendfactordivide_factorr(  r   r   r  r?  rO  r&  r'  r  s           rZ   rx   z_avg_poolnd.<locals>.fn  s   cTUBN3Z .A*WQZ7yy+a.!8!A$:KL("YYvq1F 99T1Q40Dvu{{C%%f-. &,,SWWnEM&&{{6#x#8-HH<<sH 5}EEr\   rM  )rD   r`   r:   r   r   r  r   r   rS  r  rH  rO  r^   r   r   r  r  rN  fallback_avg_pool2dfallback_avg_pool3dr  r  r7   r   rc   )r   rO  r'  r&  rP  r(  r  r   r  rh   rj  
ceil_modeshad_paddingr  rX  fallbackrx   r  r2  r   r  r?  rW  r  s    ``` ` `          @@@@@@rZ   r&  r&  E  sT    #){C0K&#&F7C(Ga###{s"""v;#w<3qzz|q#' 2222NNJJL3$E	

cTUA 3Z

 1q+vw	J

E: 7|s:.q#3?==?E{T%[(HKKME""8<<=KR!8*HAX*H}SE233
 	
 *&6"K""KER R
Y Y	F 	F$ 
		||~	
B Iu

s   -H6c                   dk7  sJ d       ssddgt        | t              sJ t        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |j                               dv sJ | j	                          |j                         ^ }t        d|      \  }	}
t        d|      \  }}| j                         d   xs d   xs |
xs || j                         ^ }t        |j                               }|j                         }t        fdt        d   dz        D              t        fdt        d   dz        D              z  }|dkD  rt        | ||      S fd	fd
}t        j                  | j                         |||      }|S )Nr   divisor must be not zeror&   r  r(   c           
   3  t   K   | ]/  }t        |d    z  t        d |d    z
  d    z        z
  d       1 ywr  r  r  s     rZ   r   z&avg_pool2d_backward.<locals>.<genexpr>  r  r  c           
   3  t   K   | ]/  }t        |d    z  t        d|d    z
  d    z        z
  d        1 ywr  r  r  s     rZ   r   z&avg_pool2d_backward.<locals>.<genexpr>  r  r  rU  c           	     @   t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  t        j
                  | |      |      }t        j                  t        j
                  ||      |      }	t        j                  t        j                  ||      t        j                  t        j                  t        j                        |            }
t        j                  t        j                  |	|      t        j                  t        j                  t        j                        |            }t        j                  |t        j                  dt        j                              }t        j                  |	t        j                  dt        j                              }	t        j                  |
t        j                  t        j                              }
t        j                  |t        j                  t        j                              }t        j
                  t        j                  |
|      t        j                  ||	            }|S )z{
        This computes the scaling factor that we will divide an element
        by when `count_include_pad=False`
        r   r(   )
rH   r;  rp   r  rk  rN  rU  r   r9  rT  )r  r  stride_hstride_wpad_hpad_wr  r  r7  wstartr8  wendr:  heightrO  r&  r'  r  s                rZ   !compute_pool_size_without_paddingz>avg_pool2d_backward.<locals>.compute_pool_size_without_padding  s   
 <<q	5;;7<<q	5;;7WQZ5WQZ5<<A<<<A<X.6X.6{{GGFH%GGCNN65;;7?
 {{GGFH%GGCNN5%++6>
 VS\\!U[[%ABVS\\!U[[%AB{{4!DE{{4u{{!CDf 5swwtV7LMr\   c                8   | ^ }}}|d   z   }|d   z   }t        j                  t        |d   z
  d   z   d         t        j                        }t        j                  t        |d   z
  d   z   d         t        j                        }t        j                  t        |d         dz   t        j                        }t        j                  t        |d         dz   t        j                        }t        j
                  |t        j                  dt        j                              }t        j
                  |t        j                  dt        j                              }t        j                  |t        j                  t        j                              }t        j                  |t        j                  t        j                              }d }t              D ]+  }	t              D ]  }
t        j                  |t        j                  |	t        j                              }t        j                  |t        j                  |
t        j                              }}nssd   d   z  }n	 ||      }t        j                   g |t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d      t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d            |      }t        j                  t        j                  ||      t        j                  ||            }|;t        j                  ||t        j                  dt        j                               }t        j                  |t        j                  ||      |      } . |J |S )Nr   r(   Fr  r8  )rH   r9  r"   rp   r  rT  r;  rU  r   r   r  rQ  rk  r>  r=  r  rR  )r   rK  r?  ri  r  r  r  r  r  r  r  r  r  rW  partrE  rL  r(  r  r  r  r>  rO  r&  r  r  r'  r  s                   rZ   rx   zavg_pool2d_backward.<locals>.fn   s2   A
N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM' *	RC]+ )RWWWcll3&DEWWWcll3&DE#/,E&k'N[^;E=b"EE{{#11 #$&s||Au{{7S(T!" !.&+  11 #$&s||Au{{7S(T!" !-&+& ). xxFF2u%FF2u% #"yytS\\#u}}5UVH"yyswwx/FQHS)R*	RV ###r\   rM  )r`   r:   r   r   r  rS  rO  r^   r   r  r   fallback_avg_pool2d_backwardr7   r   rc   )r  r   rO  r'  r&  rP  r(  r  r   _h_outrk  _w_outrm  r  r   rX  rx   r  rL  r  r  r>  rK  r  r  r  r  s     ``` ``          @@@@@@@@@rZ   avg_pool2d_backwardrR    s    #'71'<X>XX<a&k9---a###{q   v;!w<1qzz|&&&

Q%;FJ &eQVWiXFJ))+K!*F
FjFJK&1&:&:&<#Q|AJJL!HKKME {1~)* M  {1~)* M
  -/KR+	
 		
 8? ?B 
		%%'	
B Ir\   c                    dk7  sJ d       ssg dt        | t              sJ t        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |j                               dv sJ | j	                          |j                         ^ } t        d|      \  }	}
t        d|      \  }}t         d|      \  }}| j                         t              xs
 |
xs |xs || j                         ^ }t        |j                               }|j                         }fdt        d      D        \  z  z  }|d	kD  rt        | ||      S  fd
fd}t        j                  | j                         |||      }|S )Nr   rA  )r   r   r   r   )r   r   r(   r&   c              3  j   K   | ])  t        fd t           dz        D               + yw)c           
   3  t   K   | ]/  }t        |   z  t        d |   z
     z        z
  d       1 ywr  r  )r   r  rh   rO  r'  s     rZ   r   z0avg_pool3d_backward.<locals>.<genexpr>.<genexpr>  sF      
 VAYQ[^);q	(I!JJAN
r  r&   N)r  r   )r   rh   rO  r'  s    @rZ   r   z&avg_pool3d_backward.<locals>.<genexpr>  s;      3
 	 	 
;q>A-.
 	
3s   /3}   c           	        d D        \  }}}d D        \  }}}d D        \  }	}
}d t        | ||g|||g|||g      D        \  }}}d t        |||g|	|
|gg|||g      D        \  }}}d |||fD        \  }}}d t        |||gg      D        \  }}}t        j                  t        j                  t        j                  ||      t        j                  ||            t        j                  ||            }|S )Nc              3  d   K   | ](  }t        j                  |t        j                         * y wr   r  )r   r  s     rZ   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s     'UQ(D'U   .0c              3  d   K   | ](  }t        j                  |t        j                         * y wr   r  r2  s     rZ   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s     Ms||Au{{;MrY  c              3  d   K   | ](  }t        j                  |t        j                         * y wr   r  )r   r   s     rZ   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s$      (
-.CLLEKK((
rY  c              3  x   K   | ]2  \  }}}t        j                  t        j                  ||      |       4 y wr   )rH   rk  rN  )r   rK  r  pads       rZ   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s3      "
1c GGCGGAqM3'"
s   8:c           
   3     K   | ]i  \  }}}}t        j                  t        j                  ||      t        j                  t        j                  |t        j
                        |             k y wr   )rH   rU  r   r9  rp   r  )r   r  r   r   r]  s        rZ   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  sU      

 #q#s KKq!3773>>#u{{+KS#Q

s   A/A1c              3     K   | ]<  }t        j                  |t        j                  d t        j                               > ywr1  rH   rT  r;  rp   r  )r   r  s     rZ   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s2      "
 KKs||Au{{;<"
   AAc              3     K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr   rH   rU  r9  rp   r  )r   r  r   s      rZ   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s6      
S KKS^^C=>
   AA)r   rH   rN  rk  )pdr  r  stride_drE  rF  pad_drG  rH  kernel_dr  r  dstartr7  rI  dendr8  rJ  r:  depthrK  rO  r&  r'  r  s                      rZ   rL  z>avg_pool3d_backward.<locals>.compute_pool_size_without_padding  sH   'Uf'U$(HMWMue(
2=(
$(H"
 Rx8<ueU>S"


 '*(8X.&u%	'	

dD"
 &&1"

tT 2UFE4JK
dD GGCGGD&)3774+@A3774QWCX
 r\   c                .   | ^ }}}}d t        |||g      D        \  }}}d t        |||g!      D        \  }}}d t        |||g!      D        \  }}	}
d |||fD        \  }}}d t        ||	|
g g      D        \  }}	}
d }t              D ]{  }t              D ]i  }t        "      D ]W  }d t        |||g|||g      D        \  }}}}n ssd   d   z  d	   z  }n
 |||      }t        j                   g |t        j                  t        j
                  |t        j                  |t        j                  dt        j                                    d
      t        j                  t        j
                  |t        j                  |	t        j                  dt        j                                    d
      t        j                  t        j
                  |t        j                  |
t        j                  dt        j                                     d
            |      }t        j                  t        j                  t        j                  ||      t        j                  ||	            t        j                  ||
            }|;t        j                  ||t        j                  dt        j                              }-t        j                  |t        j                  ||      |      }Z l ~ |J |S )Nc              3  ,   K   | ]  \  }}||z     y wr   r   )r   r   r]  s      rZ   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s     Avq#1s7Ar  c              3     K   | ]<  \  }}}t        j                  t        ||z
  |z   |      t        j                         > y wr   rH   r9  r"   rp   r  )r   r   r   r  s       rZ   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s<      %
1a NN8AEAIq15;;?%
ra  c              3     K   | ]8  \  }}t        j                  t        ||      d z   t        j                         : ywr  ro  )r   r   r  s      rZ   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s6      
1 NN8Aq>A-u{{;
s   >A c              3     K   | ]<  }t        j                  |t        j                  d t        j                               > ywr1  r`  )r   pstarts     rZ   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s2      %
 KKQ <=%
ra  c              3     K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr   rc  )r   pend
pooled_dims      rZ   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s6      
 j KKcnnZEF
rd  c              3     K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr   )rH   r   r;  rp   r  )r   rr  p_s      rZ   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s6      "&FB R(EF"rd  r   r(   r&   Fr  r8  )r   r   rH   r  rQ  rU  rk  r;  rp   r  r>  r=  r  rR  r   )#r   rK  r  r?  ri  pdstartr  r  pdendr  r  r  pd_r  r  re  r  r  rW  rN  rE  rL  r(  d_window_sizer  r  r  r>  rO  r&  pooled_depthr  r  r'  r  s#                        rZ   rx   zavg_pool3d_backward.<locals>.fn  s   AqAaAY)@A1a%
1ay+v>%
!'

Q1Iv.
ue
%
"GW5%
!'
$'u%m\'R%
ue ' 8	VC]+ 7V / 6VC"*-$gw7#sC+"JBB (3 0*+ +AQ ?+a. P A"b" M;;#!' # 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!" !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %2*/!"  !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!"!4 7D< 88E!2CFF2u4EFr5)D  '#&99 $S%--(H$ $'99T3778T3JH#Um6V7V8	Vr ###r\   rM  )r`   r:   r   r   r  rS  rO  r  r^   r   r   fallback_avg_pool3d_backwardr7   r   rc   )!r  r   rO  r'  r&  rP  r(  r  r  _d_outceil_mode_drP  ceil_mode_hrQ  ceil_mode_wr   r  r   rX  rx   r  rL  r{  rk  r  r  r>  rK  r|  r  r  r  r  s!     ``` ``             @@@@@@@@@@@@rZ   avg_pool3d_backwardr  o  s    #'71'<X>XX<k9---a###{q   v;!w<1qzz|&&&$%JJL!VUFE&q+vw	FK ';FK 'q+vw	FK ))+Kg,K+KKK4?4H4H4J1Qm\AJJL!HKKME3
 q3/M=-  -/-?KS+	
 		
# #JU U Un 
		%%'	
B Ir\   c                   | j                         }t        |t              r|g}n|st        t	        |            }t	        |      dk(  rt        |      dv s
J d|        g S t        |      }t        t	        |            D ]_  }||   dk  r#||xx   t	        |      rt	        |      ndz  cc<   d||   cxk  rt	        |      k  rFn t	        |      dk(  r	||   dk(  r_J  t	        t        |            t	        |      k(  sJ d       |S )Nr   )r   rz  r:  zinvalid axis: r(   zreduction axis not unique)r   r`   r   r   r   r   r^   r    )r   r]  r   rh   s       rZ   _validate_reduction_axisr  7  s    ::<D$vSY
4yA~T{//H>$1HH/	:D3t9 M7Q;GCIs4y14GDG'c$i'CINtAw!|LLM z$ CI-J/JJ-Kr\   c               x  
 |t        | |      } | j                         t        t           t	        | |            }g }g g }g t        t                    D ]Q  }||v r&j                  |       |j                  |          -j                  |       |j                  |          S 
fd}r0t              }	D ]  }t        j                  j                  |	|<   ! n|}	| j                         
t        | j                         |xs | j                         | j                         ||	|      S )Nc                   t        |      t        	      k(  sJ r+t        |       t        
      k(  sJ D cg c]  }| |   	 } }t        |       t              k(  sJ d gt        |       t        |      z   z  }t        j                  t        |       t        	|            D ]
  \  }}|||<     |      S c c}w r   )r   r   r   r   )r(  reduction_indexrh   rp  r   varinner_loaderkeepdimskept_idxreduced_idxr   s         rZ   rt  z%_make_reduction_inner.<locals>.loader[  s    ?#s;'7777u:T***'/0!U1X0E05zS]***Fc%j3+??@	!% #k?"C
 	!HC !IcN	! I&& 1s   B;)rk   rf  rt  rL  rK  rg  )r   r   r    r   r  r   r   rd   r^   r   r  r  rO  r   rc   r   )r   r]  r  r   r0  
kept_sizesreduced_sizesrh   rt  r  r  r  r  r   s     `       @@@@rZ   _make_reduction_innerr  I  s'   Q::<Dc?3At<=DJHMK3t9 '9q!  a)OOAd1g&'' ' : 	&A''++HQK	& ==?L||~'81;;=++-& r\   c                     dd d fd}|S )Nr   c                   t        | |||      }t        j                  d| d|}t        |j                  j                  t              r|j                          |S )Nr]  r  r   r0  )r  re  r   )r  r8   r   r`   rb   rd  )r   r]  r  r   r   r0  r0  r  s         rZ   rZ  zmake_reduction.<locals>.inner{  sb    &"7
 !!XAXQWXKK
 NNr\   r   r   )r  r0  rZ  s   `` rZ   make_reductionr  z  s    T   Lr\   c                   |t        | |      } t        | |      }t        | j                         | j	                         f| j                         f| j                         |      S )N)rk   dtypes	inner_fnsr   r]  )r   r  r   rc   r   rO  r   )r   r]  r   s      rZ   _make_scan_innerr    sZ    QD!D||~==?$ZZ\ r\   r   c                   |t        | |      } | j                         t        | |      }| j                         }|t        j
                  t        j                  fv rt        | t        j                        } t        | ||      }t        fd|D              }t        j                  || j                         | j                               }t        j                  |t        |j                                     }t        t!        ||      |      S )Nc              3  (   K   | ]	  }|     y wr   r   r   rh   r   s     rZ   r   zmean.<locals>.<genexpr>       0a$q'0r   r'  )r   r   r  r   rp   rS  rR  r#  sum_rF   r+   r1   rc   r0   r   r^   div)r   r]  keepdimr   output_dtype
sum_resultdenomr   s          @rZ   r  r    s    Q::<D#At,D;;=Lu~~66Q$aw'J0400Ee1;;=XEeT**=*=*?%@AEC
E*L99r\   c                2  
 |d}| j                         
t        | |      }t        | |d      }|r|j                          t	        t        | |            }t        |||      }t        
fd|D              }|rt        j                  ||z
  d      }t        j                  || j                         | j                               }t        j                  |t!        |j                                     }t#        ||      }	|s|	fS |r|nt%        ||      }|	|fS )Nr(   T)r  c              3  (   K   | ]	  }|     y wr   r   r  s     rZ   r   z var_mean_sum_.<locals>.<genexpr>  r  r   r   r'  )r   r  r  rd  squarerk  r  rF   r   r5  r+   r1   r   rc   r0   r   r^   r  r  )r   r]  
correctionr  return_meanx_meandiffsr  r  x_varr   s             @rZ   var_mean_sum_r    s    
::<D#At,D!T4(F3q&>"EeT7+J0400E		%*,a0e1;;=XEeT**=*=*?%@AE
E"ExVGFD$9F&=r\   c                    t        | |      }t        | ||d d       }|d   }t        |d         }t        |t        j
                        xr, t        |      t        j                  k  xr t        |      dk7  S )Nr  rK  rg  r(   )	r  r  rF   r`   r   r  r   r)   rr  )r   r]  r  r   rK  reduction_numels         rZ   use_two_step_variancer    s|    #At,D"	wd$F HF#F+=$>?O?EMM2 	' 6#E#EE	'&!Q&r\   c                  dt        | ||d d       }|j                  d      }|j                  d       |j                  d       t        j                  j                  d|fd| j                         d|\  }}}	|j                          | j                         | j                         t        | |      }t        fd|D              d	 fd
}
 t        |
      |      }|r|j                          ||fS |fS )Nr(   r  rL  rf  rt  welford_reduce)r  r  r   c              3  (   K   | ]	  }|     y wr   r   r  s     rZ   r   z$var_mean_welford_.<locals>.<genexpr>  s     1q471r   c                    t        | t        j                        rD| j                  s8t	        j
                  t	        j                  | t        j                        |      S t	        j                  | |      S r   )
r`   r   r   	is_numberrH   r   r9  rp   r:  r;  r)  s     rZ   get_constant_or_index_exprz5var_mean_welford_.<locals>.get_constant_or_index_expr  sG    a$Q[[<<q%++ >FF||Au%%r\   c                           }       }t        j                  d      }| t        j                  |||z
        z  S r  )rH   r;  rT  )rb   cNzeror  r   r  rnumels       rZ   r  z#var_mean_welford_.<locals>.scale_fn  sE    &z59&vu5||Au%ckk$A...r\   r   )r  rc  r+   WelfordReductionr   r   rd  r   r  rF   r[  )r   r]  r  r  r  r   rt  r  m2r   r  r  r   r  r  r   s     `         @@@@rZ   var_mean_welford_r    s    
"	wd$F ZZ
#F
JJ{
JJ{%%,, )'kkm 	KD"a JJLKKME::<D#At,D1D11F&
/ #.
"2
&CDy6Mr\   c                   | j                         t              }t        | |d      } t        | ||||      }t	        | ||      rt        di |n
t        di |}t        fd|D              }|s|d   S |S )NFr  )r   r]  r  r  r  )r]  r  c              3  :   K   | ]  }t        |d         yw)Fr  N)r   )r   r   rf  s     rZ   r   z#var_mean_helper_.<locals>.<genexpr>  s     F!8Ayu55Fs   r   r   )r   r   r   r   r  r  r  r   )	r   r]  r  r  r  compute_dtyper   r  rf  s	           @rZ   var_mean_helper_r    s    I))4MM.A
F !w? 	(( 
 FvFFF'6!93V3r\   )r  r  c               "    t        | |||d      S )NFr]  r  r  r  r  r   r]  r  r  s       rZ   var_r    s    	W% r\   c               "    t        | |||d      S )NTr  r  r  s       rZ   var_meanr    s    	W$ r\   c                   |dk  r!t        t        j                  |       | |      S |dk(  rt        j                  d|      S |dk(  r| S t        | |dz  |      }t        j                  ||      }|dz  dk(  rt        j                  ||       }|S )Nr   r(   r&   )pow_recursiverH   rS  r;  rN  )r   r  r   r0  s       rZ   r  r  $  s    1uS^^A.E::Av||Au%%Av1a1fe,FWWVV$F	A!|#Mr\   c                .    t        j                  | |      S r   )rH   powr   r  s     rZ   
pow_nativer  3  s    771a=r\   )r   c                v    t        t              r#t              k(  rt         t                    S t        t              rdk(  rt	               S t        t              rdk(  rt               S t        d  fD              }t        |      }t        t              xr dcxk  xr dk  nc xs	 |xr dk\  }|rZ j                          fd}t        j                   j                          j                         | j                               S t         t              r: dk(  rt        d      S  d	k(  r$t!        j                               rt#              S |rDt         t              rt%               S t        t              rt'               S t)               S t+               S )
Nr  r(   c              3  r   K   | ]/  }t        |t        j                        s|j                          1 y wr   )r`   r+   r:   r   r  s     rZ   r   zpow.<locals>.<genexpr>K  s#     N1*Q2MNs   77i    r   c                F    t         |       j                               S r   )r  r   )r   r   r  rt  s    rZ   rx   zpow.<locals>.fnU  s     a??r\   rM  r&   )r`   r#  r   r  sqrtrw  r-  r   rO  r7   r   rc   r   r   r   r  r   exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorr  )r   r  r   is_integer_powembed_exponentrx   rt  s   ``    @rZ   r  r  A  sx   !USV1c!f~	Au	!s(Aw	As	QQx NANNE%e,N  3' a"32AF  	@ <<>++-::<	
 	
 !V6Q?"6nQ[[]37Na &q!,,6"-a33-a33ar\   c                   t        | t              r| j                  }n| }t        |t              r|j                  }t        |t        j                        swt        j                  | j                         | j                         |j                         | j                               j                  }t        |t        j                        sJ t        |t        j                        rg|j                         sW|j                         sGt        |j                  t        j                        s#|j                          |j                  |_        | S t        j                  j!                  |||       | S )NrM  unsafe_alias)r`   r:   rb   r+   r  r7   r   rc   r   rO  r   is_input_bufferis_module_buffer	NopKernelrd  r  realize_into)changedr  r  changed_datas       rZ   r  r  q  s   '9%||#y!hhc2==)%%'##%__&##%	

 $ 	 #r}}---,.$$&((*l''6 	HH!!..\ /  Nr\   c                .    t        | t        | |            S r   )r  r  )r   r(  s     rZ   r%  r%    s    Q	!Z011r\   c                    | |u r| S t        || j                               }t        || j                               }t	        || j                               }t        | |      S r   r  rc   r   r   r  r   r  )r  r  r  s      rZ   r  r    sR    
cz

C)
*C
3
(C
clln
%CS#r\   c                .    t        j                  | |      S r   )rH   floordivr  s     rZ   r  r        <<1r\   c                .    t        j                  | |      S r   )rH   r1  r  s     rZ   r1  r1    r  r\   c                >   t        |       xr t        |      }t        |       xr t        |      }|dk(  r,|rJ d       |rt        | |      S t        t	        | |            S |dk(  r,|rJ d       |rt        | |      S t        t	        | |            S t	        | |      S )Nr  z5floordiv operands can not be boolean at the same timer  z5truncdiv operands can not be boolean at the same time)r   r   r  r  r  r1  r  )r   r  rounding_modeboth_integerboth_booleans        rZ   div_moder    s    "1%</!*<L"1%</!*<L X!XX!-x1~C5Q3CCX!XX!-x1~C5Q3CCq!9r\   c                    t        |       xr t        |      }|rt        | |      S t        t        j                  j
                        } t        |      | |      S r   )r   logical_andr5   r  rN  r  r[  )r   r  	both_boolrx   s       rZ   rN  rN    sN    "9q'9I1a  **+!~b!!Q''r\   c                f   t        | t        j                        rt        | j                        S t        | t        j
                        rt        | j                               S t        | t        j                        r| S t        | t        j                        syt        j                  j                  j                  | j                               }t        j                  |      5  t!        j"                  t        j$                  dd      5   | j&                  | j)                          }ddd       ddd       t        t        j                  j*                  j,                        sJ t        |j.                  t        j                        r|j.                  S y# 1 sw Y   rxY w# 1 sw Y   vxY w)z:Try convert an arbitrary IR node into an ir.Constant valueNallow_indexingT)r`   r+   
MutableBoxget_constant_valuerb   r!  r"  r   Loopsrp   	_inductorops_handlerExtractConstantsHandlerrc   rI   set_ops_handlerr   objectrB  rL  inner_fn_argsvirtualizedOpsValuer   )r   r  rf   s      rZ   r  r    s/   
 !R]]#!!&&))!R[[!!!--/22!R[[! a"oo))AA!,,.QG	'"-R&&(8$?- ajj!//+,	- - c5??66??@@@#))R[[)yy- - - -s$   &&F'F*F'F$	 F''F0c                2   t        d | |fD              }|rt        | |      S t        |      x}T|j                  dk(  r*t	        j
                  t        d      |j                        }nd|j                  z  }t        | |      S d } t        |      | |      S )Nc              3  L   K   | ]  }t        |      xs t        |        y wr   )r   r   r  s     rZ   r   zdiv_prim.<locals>.<genexpr>  s"     O1oa(>OA,>>Os   "$r   infrp  c                 &    t        j                  |  S r   )rH   r  r  s    rZ   rx   zdiv_prim.<locals>.fn  s    {{D!!r\   )	r  r1  r  r   mathcopysignr#  rN  r[  )r   r  is_integralr2  rS  rx   s         rZ   div_primr    s    OAOOK1~%a((5==AuU|W]]CJw}},J1j!!" >"a##r\   c                Z    t        | |ft        j                        \  } }t        | |      S r!  )r2  r   INT_TO_FLOATr  r  s     rZ   r  r    s0     	
A$C$P$PDAq Aq>r\   c                h    t        |       xs t        |       }|rd }nd } t        |      | |      S )Nc                .    t        j                  | |      S r   )rH   modr  s     rZ   rx   zfmod.<locals>.fn  s    771a= r\   c                .    t        j                  | |      S r   )rH   fmodr  s     rZ   rx   zfmod.<locals>.fn  s    88Aq>!r\   )r   r   r[  )r   r  r  rx   s       rZ   r  r  	  s9    !!$:(:K	!
	" >"a##r\   c                   t        | j                               st        | j                               r|t        j                  }t        d|      } || |||      S )Nr  rv  r   r   r   r   rp   r:  r  r   r]  r  r   rx   s        rZ   r  r    sK     	'+;AKKM+J
-	U	;Baxu--r\   c                   t        | j                               st        | j                               r|t        j                  }t        | j                               dk(  r(|dv sJ |xs | j                         }t        | |d      S d }t        | ||      }t        j                  j                  d	i |d|i\  }|t        | ||      S |S )
Nr   r   r:  Tr  c                @    | \  }|\  }t        j                  ||      fS r   )rH   r   a_tupleb_tupler   r  s       rZ   
combine_fnzcumsum.<locals>.combine_fn8  #    1r\   r]  r   r  r   r   r   )r   r   r   rp   r:  r   r   r   r  r+   Scanr   fallback_cumsumr   r]  r   r  r   r0  s         rZ   cumsumr  ,  s     	'+;AKKM+J
-
1::<Aw&5t,, 
 ad%8F??J?IV~qd%88Mr\   c                   t        | j                               st        | j                               r|t        j                  }t        | j                               dk(  r(|dv sJ |xs | j                         }t        | |d      S d }t        | ||      }t        j                  j                  d	i |d|i\  }|t        | ||      S |S )
Nr   r  Tr  c                @    | \  }|\  }t        j                  ||      fS r   )rH   rN  r  s       rZ   r  zcumprod.<locals>.combine_fnP  r  r\   r  r  r  r   )r   r   r   rp   r:  r   r   r   r  r+   r  r   fallback_cumprodr  s         rZ   cumprodr!  D  s     	'+;AKKM+J
-
1::<Aw&5t,, 
 ad%8F??J?IV~t599Mr\   c                
   d }| j                         }t        | j                               dk(  r|dv sJ t        |       S t	        | ||      }t        j                  j                  di |d|i\  }|t        | |      S |S )Nc           	     *   | \  }|\  }t        j                  ||      }t        j                  ||      }||k7  t        j                  |       z  }t        j                  |t        j
                  t        j                  ||z
              |z   |      fS r   )rH   rU  rT  r  r  log1pexp)r  r  r   r  min_vmax_vrE  s          rZ   log_add_exp_helperz(logcumsumexp.<locals>.log_add_exp_helper^  s{    Aq!Aq!CIIe$4#45		$		#''%%-*@ AE I1MOOr\   r   r  r  r  rG  r   )	r   r   r   rw  r  r+   r  r   fallback_logcumsumexp)r   r   r(  r   r   r0  s         rZ   logcumsumexpr*  \  s    P KKME
1::<Ag~~Qxac7FGG4FGIV~$QC00Mr\   c                   t        | j                               dk(  r,|dv sJ t        |       t        | t        j
                        fS | j                         }t        j                  d|d      }t        | ||      }|t        j
                  f|d<   | j                         d	 f|d
<   t        j                  j                  di |d|i\  }}|t        | |      S ||fS )Nr   r  r   rh  Fr   arg_break_ties_leftr  r  c                     yNrindexr   rV  s    rZ   r7  zcummax.<locals>.<lambda>      r\   r  r  rG  r   )r   r   rw  r  rp   r:  r   r+   get_reduction_combine_fnr  rO  r  r   fallback_cummaxr   r]  r   r  r   r   r  s          rZ   cummaxr5  r      
1::<AwQxAU[[999KKME,,5J ad%8Fu{{+F8==?,>?F;ggnnEvE*EOFG~qd++7?r\   c                   t        | j                               dk(  r,|dv sJ t        |       t        | t        j
                        fS | j                         }t        j                  d|d      }t        | ||      }|t        j
                  f|d<   | j                         d	 f|d
<   t        j                  j                  di |d|i\  }}|t        | |      S ||fS )Nr   r  r   argminFr,  r  r  c                     yr/  r   rV  s    rZ   r7  zcummin.<locals>.<lambda>  r1  r\   r  r  rG  r   )r   r   rw  r  rp   r:  r   r+   r2  r  rO  r  r   fallback_cumminr4  s          rZ   cumminr;    r6  r\   c                   t        | j                               st        | j                               r|t        j                  }t        d|      } || |||      S )Nr  rv  r   r  r  s        rZ   r  r    sK     	'+;AKKM+J
-	e	<Baxu--r\   c                ^    t        | t        j                        }  t        d      | ||      S )Nr  r]  r  )r   rp   r   r  r   r   r  s      rZ   
reduce_anyr@    s(    EJJA >% w??r\   c                Z    |t        | ||      t        | ||      fS t        | d |      S Nr>  )reduce_amaxreduce_argmaxr?  s      rZ   
reduce_maxrE    :    
g6!#8
 	

 qtg66r\   c                Z    |t        | ||      t        | ||      fS t        | d |      S rB  )reduce_aminreduce_argminr?  s      rZ   
reduce_minrJ    rF  r\   xor_sumr  r  rh  rv  r8  
logical_or)rW  rJ  stabler   
descendingc          	        |d}| j                         }| j                         }t        t        |      |      }t        |      dk(  r't	        |       t        d|t        j                  |      fS t        |      r||   nd}t        j                  j                  j                  |t        j                  t        j                        j                        st        | |||      S t!        |ddt        j                  |d      }dgt        |      z  }t        |      r|||<   t#        ||      }t%        ||      }t&        j(                  j+                  || j,                  |j,                  f| j/                         |j/                         f||||      \  }	}|	t        | |||      S |J |	t1        |t        j                        fS )NFr   r(   rM  )r  r  r   rk   r  )rk   r  r  r   r]  rN  rO  )r   rc   r   r   rw  r)  rp   r:  rI   rR   r  statically_known_ltr  int16r  sort_fallbackr  r  r  r+   Sortr   r   rO  r   )
r   rN  r   rO  r  rk   r  r  
view_shaper   s
             rZ   sort_stablerV    s   ~JJLE\\^F
3u:s
+C
5zQQxq&%++u=== ZuSzQH77//%++ekk:R:V:VWQv3:NNVSXG s5z!J
5z"
37J'GWe$Gggnn'==?G$7$7$9: % OFG ~Qv3:NN8GU[[111r\   c                     t        | d||      S )NFrM  )rV  )r   r   rO  s      rZ   sortrX    s    qCJGGr\   c                <    t        | |t        j                  |      S )Nr   r   rY  r  r   r  )rU   r   rY  s      rZ   register_pointwise_numericr\    s!    
;HH'	 r\   c                :    t        | t        j                  d      S )NT)r   r  r[  r  s    rZ    register_pointwise_numeric_ldf64r^    s    
;HH" r\   r  logical_not)rJ  )r   r   r0  identity)rb  pointwise_overrides_datac              #  F  K   t         |   t        | j                  d       }|y fd}t        |t        j
                  j                        r9|j                         D ]%  }t        ||      }|j                   ||      f ' y |j                   ||      f y w)Nc                4    j                   t        |       S y r   )tritonr  )rU   rb   s    rZ   make_triton_fallbackz6_get_pointwise_overrides.<locals>.make_triton_fallbacku  s    ;;#B'' r\   )	ra  r   r   r`   rp   rz   r   r   r   )nsr   rU   re  olnamer2  rb   s         @rZ   _get_pointwise_overridesrh  o  s     #D)D	TYY	%B	z( "ejj112lln 	IFV$Bd..0DR0HHH	I $**,@,DDDs   BB!rZ  c                d    | t         |<   t        j                  |        fd}t        | |       y )Nc                      | i |}g }t        | d   |      D ]"  \  }}|j                  t        ||d             $ |S )Nr   Tr  )r   rd   r  )ri   r   resultsmut_resultsr   r0  outplace_ops         rZ   rx   z$register_foreach_inplace.<locals>.fn  sV    t.v.tAw0 	JKCyf4HI	J r\   )rO   r`  r   r   )aten_opoutplace_aten_oprm  rx   s     ` rZ   register_foreach_inplacerp    s.    07,-G$ w+r\   c                2    t        | d       fd       }|S )Nry  c                 l     | i |}t        || d   j                               }t        | d   |      S r  )r   r   r  )ri   r   r0  rm  s      rZ   rx   zregister_inplace.<locals>.fn  s<    d-f-&$q'"3"3"56a&))r\   )r  )rn  rm  rx   s    ` rZ   register_inplacers    s#    wD9* :*
 Ir\   c                     y r   r   r  s      rZ   sym_constrain_rangeru  
      r\   c                    t         j                  j                  j                  d   }t	        |t
        j                        sJ |j                  j                  S r  	rI   rR   rS   rT  r`   rp   r  rX   r  r   r   r  s      rZ   sym_sizerz    s@    
''


#
#E
*C c5<<(((88==r\   c                    t         j                  j                  j                  d   }t	        |t
        j                        sJ |j                  j                  S r  rx  ry  s      rZ   
sym_strider|     s>    
''


#
#E
*Cc5<<(((88==r\   c                "    | j                         S r   )rT  )r   s    rZ   	sym_numelr~  (  s    ;;=r\   c                &    t        j                  |  S r   )r   Addr  s    rZ   sym_sumr  1  s    99dr\   c                    t        d      )NzHelpful for debuggingr   )r  ri   r   s      rZ   foobarr  6  s    
5
66r\   c                8    | j                          t        |       S r   )rd  rw  r   s    rZ   _realizer  ;  s    IIK8Or\   c                R    | j                          t        j                  | |       | S r   )rd  r+   ResizeStorageBytes)variabler  s     rZ   resize_storage_bytes_r  A  s#    (H-Or\   c                    | j                          |j                          t        j                  t        j                  | |            S r   )rd  r:   r   r+   SetSourceTensorKernel)r  source_tensors     rZ   set__source_tensorr  H  s5    LLNB44T=IJJr\   r  c                    | |u r| S t        || j                               }t        || j                               }t	        || j                               }t        | |      S r   r  )r  r  s     rZ   
fsdp_copy_r  Q  sR    #:JS^^-.sCMMO,S#,,.)c""r\   c                 	
 t        | t              sJ t        |t        t        f      sJ |t        j
                  }|t        j                  k(  rt        d|       |t        j                  k(  rt        |      dk(  sJ |t        j                  k(  rt        |      dk(  sJ | j                         
| j                         }| j                         }t        | j                  t        j                         r| j                  j#                         | _        t	        j$                         rit        j&                  j(                  j*                  rEt-        |      rt/        d      n0t1        |      r t	        j2                  |      j4                  ndndt6        j8                  j:                  j=                  
d      rt?        |||      S tA        | 
gd	g      }|jC                         	t        jD                  jG                  ||      }t        jH                  ||||      jK                         	
fd
}tM        jN                  |||t        |            }|S )Nzunsupported memory format: r   r   nanTr8  r   r  r(   c                    |       t        j                  t        j                        }t        j                  t        j                        }t        j                  ||      }t        j
                  |fd      S )Nc                       g      S r   r   )
flat_indexflat_loaders   rZ   r7  z*resize.<locals>.inner_fn.<locals>.<lambda>  s    ZL(A r\   )rH   r9  rp   r:  r=  r?  )	r   flat_index_exprlimitrE  r  r  	old_numelout_indexeruninitalized_vals	       @rZ   rL  zresize.<locals>.inner_fn  sZ     %
..U[[Ay%++6vvou-zz$ ACSTTr\   rM  )(r`   r:   r^   r   rp   contiguous_formatpreserve_formatr0  channels_lastr   channels_last_3drT  r   r.  rb   r+   r!  r"  r  r   r  fill_uninitialized_memoryr   r#  r   r  r  rI   rR   r  r  rN  r,  rO  rB   stride_ordered_for_memory_formatr%  rD  r7   r   )r   r   r  r   rk   x_flat
out_striderL  rf   r  r  r  r  s            @@@@rZ   resizer  \  s   a###dT5M***//---8HII+++4yA~~...4yA~~IKKME""$F!&&"++&##% 	224KK%%??% $U|e$${{5155# ww//	1=D*%GG		
 	
F $$&K""CCD-XJ..jANNPKU 

UXd4jC Jr\   )auto_functionalizedc                    ddl m} |j                  |      }t        j                  | ||i ||       |j                         D ci c]  \  }}t        |t              s|| c}}S c c}}w )Nr   )kernel_side_table)
kernel_idxgridtma_descriptor_metadatakernel_args)*torch._higher_order_ops.triton_kernel_wrapr  get_constant_argsr+   UserDefinedTritonKernelr   r`   r:   )	r  constant_args_idxr  r  r   r  constant_argsr  r  s	            rZ   triton_kernel_wrap_r    sj     M%778IJM 7/v//	 &,\\^Rcz#y7QCHRRRs   A*!A*c                ^   t        d | g|D              rTd}t        j                  j                  j                  j                  dd       x}r| d| }|t        j                  _        t        j                  j                  | |||      }t        t        t        j                  |            S )Nc              3  V   K   | ]!  }t        |t              xr t        |       # y wr   r8  r  s     rZ   r   zcond.<locals>.<genexpr>  s#     
Ma:a 1Yq\1
Mr9  z"control flow operator: torch.cond.r  r  )r  rI   rR   rS   rT  rU  r  r+   Conditionalr   r^   mapr:   )predtrue_fnfalse_fnoperandsr   r  r0  s          rZ   r   r     s    

MD;L8;L
MM2''..3377tLL;LE*;-8C,/)^^""4(HEFI$$f-..r\   c                ^   t        d ||z   D              rTd}t        j                  j                  j                  j                  dd       x}r| d| }|t        j                  _        t        j                  j                  | |||      }t        t        t        j                  |            S )Nc              3  V   K   | ]!  }t        |t              xr t        |       # y wr   r8  r  s     rZ   r   zwhile_loop.<locals>.<genexpr>  s+       	1f.)A,.r9  z(control flow operator: torch.while_loop.r  r  )r  rI   rR   rS   rT  rU  r  r+   	WhileLoopr   r^   r  r:   )cond_fnbody_fncarried_inputsadditional_inputsr   r  r0  s          rZ   
while_loopr    s    
 "33  9''..3377tLL;LE*;-8C,/)\\  '>CTUFI$$f-..r\   c                    t         j                  j                  | |      }t        t	        t
        j                  |            S r   )r+   InvokeSubgraphr   r^   r  r:   )subgraph_fn
identifierr  r0  s       rZ   invoke_subgraphr    s2    %%k8<FI$$f-..r\   )schemec                  d }t         j                  j                  j                  j	                  dd       }|J t        | j                  j                  j                        D ]  \  }}|j                  dk(  r!||   t         j                  j                  |<   7|j                  dk(  rt         j                  j                  |      \  }}t        j                  ||j                               D ]  }	|	j                          |j                  r7t         j                  j                   j#                  |	j%                                t         j                  j&                  j#                  |	j%                                 t(        j*                  j,                  j/                  t         j                  |||      }Xt         j                  j1                  |      t         j                  j                  |<    |S )Nquant_optionsplaceholderr  )rI   rR   rS   rT  rU  r_   r  nodesrU   envfetch_args_kwargs_from_envr   r   r   rd  codegen_low_precisionlow_precision_codegen_opsr   re  invoke_quant_opsrp   r  Interpreterr  run_node)
r  r  r  r  r  rh   rX   ri   r   r   s
             rZ   invoke_quant_tracerr    sl   FGG((--11/4HM$$$[55;;AAB 7477m# (AGGKKWW 77==dCLD&__T6==?; E		 66GG5599!:N:N:PQ((,,Q-A-A-CDE XX))00$fMF ! 0 0 6AGGKK%7( Mr\   c                  
 ddl m}m} t        |      dkD  rt	        d      t        j                  ||      D cg c](  } ||j                         |j                               * }} || |      

fd}t        |d   dd       }t        d |D              |d	<   t        d
 |D              |d<   t        j                  j                  d|dd|}	|	d   t	        d      |	S c c}w )Nr(   )InputDescriptorlower_pointwise_subgraphr   zSUnable to generate code for associative_scan op, because there are lifted argumentsr  c                d     g t        j                  |       t        j                  |       S r   )r  r  )lhsrhslowered_combine_fns     rZ   wrapped_combine_fnz,associative_scan.<locals>.wrapped_combine_fn  s6    ! 
$
$
 	
r\   r  c              3  <   K   | ]  }|j                           y wr   )r   r  s     rZ   r   z#associative_scan.<locals>.<genexpr>  s     7qQ[[]7   r  c              3  <   K   | ]  }|j                           y wr   r  r  s     rZ   r   z#associative_scan.<locals>.<genexpr>  s     <A<r  r  F)r  can_fallback_to_atenz/Unable to generate code for associative_scan opr   )r~  r  r  r   r0  r   r   r   rc   r  r   r+   r  r   )r  xsr  r  r  r   subgraph_inputsr  r   r0  r  s             @rZ   associative_scanr    s     M
!a
 	
 R( 	akkmALLNCO  2*oN
 be!48F7B77F8<<<F;WW^^ %" F
 ayLMMM-s   -C'c                     y r   r   )tokenss    rZ   _sink_tokensr    rv  r\   c                Z   t        j                  j                  |g|i |}ddlm}  ||||      }|J t
        j                  j                  |   }||fS t        j                  t         j                  t        j                  |      }t        |t        t        f      s||fS |g|S )Nr   )get_effect_key)r+   EffectfulKernelr   torch._higher_order_ops.effectsr  rI   rR   effectful_opsr  r  MultiOutputr:   r`   r^   r   )tokenrU   ri   r   r0  r  effect_typeeffectful_kernels           rZ   with_effectsr    s    &&r;D;F;F> T62K"""ww,,[9~ ""!!"..)2B2BFKFftUm, &)) *6**r\   )register_comm_loweringsc                x   t        | |ddd      }|d   }t        j                  j                  j	                  t        |            }t        j                  j                  di |d|d\  }}|dk(  rZt        j                  j                  j                  |      t        j                  k\  r t        j                  d| d|d	|\  }}||fS t        j                  t!        j"                  d
             t%        | |d      }	t'        t(        j*                     t-        | |	            }
t/        |
|d      }|	|fS )zn
    Lowering inductor_prims.prepare_softmax_online to compute max/sum in one pass if no split is needed.
    TNr  rg  online_softmax_reduce)r  r  r(   r&   )re  
num_outputreduction_hintz
            Online softmax is disabled on the fly since Inductor decides to
            split the reduction. Cut an issue to PyTorch if this is an
            important use case and you want to speed it up with online
            softmax.
            )r  r   )r  rI   rR   r  simplifyrF   r+   r8   
num_splitsr  r)   rr  r4   r   r  r  textwrapdedentrC  rM   r  r%  rk  r  )r   r   r   rg  r  hint	num_split
max_tensor
sum_tensorr  r%  xsums               rZ   prepare_softmax_onliner  9  s4   
 #	d$dF 01WW&&}5E'FGFll-- 
.OD) 	QGG&&v.&2T2TT!7!>!> "
Qt"
?E"

J :%% 	OO		
 1cD1!#a,/Ct,Tzr\   )r   )quantized_lowerings)mkldnn_lowerings)jagged_loweringsc              #  X  K   t        | t        j                  j                        sJ d       t        j                  |       }	  t        |       t        |              d |r
|t        | <   yt        j                  |        y# |r
|t        | <   w t        j                  |        w xY ww)z^
    A context manager to force fallback an op. Used in unit test
    for FallbackKernel.
    z+Only OpOverload to make the clean up easierN)	r`   rp   rz   r{   rM   rU  r  r  rc  )rU   old_handlers     rZ   force_fallbackr    s      b%**//0 50 --#K".r23'IbMMM" 'IbMMM"s   AB*B ""B*#B''B*)re   z%Iterable[Union[tuple[Any, Any], Any]])rx   zCallable[..., Any]r  zOptional[Callable[..., Any]])r   r   )r   r   )ri   z	list[Any]r   zdict[str, Any]r   r   r   )Optional[ELEMENTWISE_TYPE_PROMOTION_KIND]r   r   r  z tuple[list[Any], dict[str, Any]])r   r  )r   r  r  z.Callable[[Callable[_P, _T]], Callable[_P, _T]])NN)NNNNFN)F)r   r:   r   torch.dtype)r   r:   rk   torch.devicer   )r   r   l            r(   Trz  )ri  r:   ra  r:   rc  r:   r]  r   r`  r   r_  r   r   r  r  r:   )ri  r:   ra  r:   rc  r:   r]  r   r`  r   r_  r   r   r  rf  Optional[torch.dtype]r  r:   )ri  r:   rW  r#  rX  r   r`  r   r_  r   r   r  r  r:   )ri  r:   rW  r#  rX  r   r`  r   r_  r   r   r  rf  r  r  r:   )ri  r:   rW  r:   rX  r:   r`  r   r_  r   r   r  r  r:   )ri  r:   rW  r:   rX  r:   r`  r   r_  r   r   r  rf  r  r  r:   )r   r   r(   )r  r   r  r   r  r   r  )T)rl   ztorch.Tensor)rX   ztorch.fx.Node)NTF)rk   r  )r   	list[int]rE  r:   r  r_  r  r   )
r  r   r  r   r   r  rE  r:   r  r   )r  r:   r  z.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr])r  r:   r  ztuple[str, sympy.Expr])r  r:   r  r:   r  r   r  r   r  Optional[str]r  Optional[TensorBox]r  r:   )ri  r:   r  r:   r  r   r  r   )r   r   r(  r   )r   NNr(   )NNN)r:  FF)r   r   )r"  r  r   r   r  r	  r  r   )r   r   r  r	  )r   r   r  r   )r&   F)r  ztuple[Optional[float], ...]r  r   r  r   )ra  Optional[float])r
  r  r  r  )r  r  r
  r  r  r  )r   r:   r&  zSequence[int]r(  r#  r  r
  )rh   
sympy.Exprr  zUnion[sympy.Expr, int])rh   r  r  r  )rh   r  r  r  r  r  )Nrp  N)Nr   r(   F)r8  )NNNN)r   r   FTN)r  rJ   r   )r   r  r  zOptional[ir.Constant])r:  F)r  ir.Subgraphr  r_  )r  r  )r  r  r  ztuple[torch.Tensor]r  (  
__future__r   
contextlibrD  r   r   loggingr  r  r%  r  r  collectionsr   collections.abcr   r   typingr   r   r	   r
   r   r   r   typing_extensionsr   unittest.mockr   r   rp   $torch.ao.quantization.fx._decomposedtorch.fxtorch.utils._pytreer   _pytreer  torch._dynamo.utilsr   (torch._higher_order_ops.associative_scanr   r  r   torch._prims_commonr   r   r   r   r   r   r   r   r   r   r   torch.fx.experimental.sym_noder   r   torch.utils._ordered_setr    torch.utils._sympy.functionsr!   r"   r#   r$   r%   _dynamo.utilsr'    r)   r*   r+   r,   decompositionr-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   r  rH   rI   r  rJ   rK   rL   FALLBACK_ALLOW_LIST	getLoggerr  r-  rM   __annotations__rN   rz   r{   r  r  tr_c10dr  r   _higher_order_opsr  rW   r`  rO   quantized_decomposedr[   rm   r}   r|   r   r   r   r,  r1  r  rR  bmmconvolutionconvolution_backwardr  r  r  r  r  _int_mmr  r  rR  r  r:  rS  rR  r?  	complex32	complex64r   rR  r   r   r   r   r   r   r   r   r   r	  r,  r  r   r2  r[  rq  r   r  r  r  r  r   r  r  
device_putr  r  r  r  r  r   aliasdetachdetach_liftview_ofr  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  _unsafe_viewreshaper  slicer  r.  rI  quantize_per_channelre  dequantize_per_channelrm  quantize_per_tensorrr  dequantize_per_tensorru  r6  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  	lru_cacher  r  r  r  r3  r8  rngprimsrH  rJ  rO  	bernoullirK  rS  rW  r[  r^  r;  rd  ra  rb  ri  rh  rg  randintforce_stride_orderrp  rE  rr  rz  ru  lookup_seedr{  randomr  r  r  r  r  rP  r  r  r  r  r  rt   r  _adaptive_avg_pool3dadaptive_max_pool3dfractional_max_pool3dmax_pool3d_with_indicesuniformexponential_pdist_forwardsoft_margin_loss_backward_cdist_forward_cdist_backward
_trilinearsegment_reduce_segment_reduce_backwardhistc	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsaddbmm_addmm_activation
_cudnn_rnn_cudnn_rnn_backward_embedding_bag_embedding_bag_forward_only_embedding_bag_backward*_embedding_bag_per_sample_weights_backward_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functional max_pool3d_with_indices_backward_adaptive_avg_pool2d_backward_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backwardreplication_pad1d_backwardreplication_pad2d_backwardupsample_linear1d_backwardupsample_bicubic2d_backwardupsample_trilinear3d_backwardgrid_sampler_2d_backward_pdist_backwardrX  rN  kthvaluetopkr  median	nanmedianr  resize_
resize_as__linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svd	lu_unpackormqr_linalg_check_errorslinalg_pinvatol_rtol_tensor_linalg_eightriangular_solvelinalg_cholesky_excholesky_inversecholesky_solvegeqrf_fft_r2cnonzerogcd_thnn_fused_lstm_cell_prims	rng_primsrun_and_save_rng_staterun_with_rng_stategraphsafe_run_with_rng_statemasked_scattermasked_scatter_backwardr  angle_efficientzerotensor(_sparse_coo_tensor_with_dims_and_tensors	to_sparse
_to_sparser   r/  r  r  #_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward#_scaled_dot_product_cudnn_attention,_scaled_dot_product_cudnn_attention_backward+_scaled_dot_product_flash_attention_for_cpu4_scaled_dot_product_flash_attention_for_cpu_backward0_scaled_dot_product_fused_attention_overrideable9_scaled_dot_product_fused_attention_overrideable_backward_flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardindex_reducerx  rw  r  r  r  r  r  r  scalar_tensorr  
LongTensorr  r  r  r!  r)  r  r-  r  r,  r  r  r  
zeros_liker>  rB  r5  rH  rL  rN  rV  rd  rk  r}  r  r  r(  r  r  r  r  r  r  r  r  r  fallback__unsafe_masked_indexr  ,fallback__unsafe_masked_index_put_accumulater  r  r  r  r  r  r  r  r  r  r  r  r  r  rQ  r  r.  r  r5  r6  rC  rH  rS  rY  r^  rp  rs  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  r;  r+  r<  r&  rO  r  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Tensor_Tensorr  Scalarr  Tensor_Scalarr  r  r%  r  r  r1  r  r  rN  r  r  true_divider  r  r  r  r  r  r!  r   r*  r)  r5  r3  r;  r:  r  r  r@  r  rE  r  rJ  rK  r  rC  r  rH  rh  rD  r8  rI  r   rS  rV  r\  r^  rsqrtr%  r  expm1relur  r  r  rk  cossinabsbitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erfr$  tantanhr  r_  rL  logical_xorrT  rU  	clamp_min	clamp_maxnegrS  	remaindersignsignbit	_neg_viewler=  r<  r  r  necoshsinhacosacoshasinasinhatan2atanatanhr  erfcerfinvhypotlog10log2	nextaftercodegen.commonrb  ra  rh  r   rU   r   rY  _foreach_addListforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_rsqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_copyrp  _foreach_add__foreach_mul__foreach_div_rs  add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__ru  rz  r   r|  r~  r   methodfuncr  r  _inductor_testrd  r  r  r  set_source_Tensorr  fsdpr  r  *torch._higher_order_ops.auto_functionalizer  r  higher_orderr   r  r  invoke_quantr  r  r  r  comm_loweringr  r  r   r  register_quantized_opsregister_woq_mm_opsr  register_onednn_fusion_opsr   register_jagged_opscontextmanagerr  r   r\   rZ   <module>r"	     sd5   "        	   # . O O O '    +  $ $ ( H U    M /  - 8 8 =          * T]t_ !   g!FH	C H     .Juzz,,-/	yy~~
))

		9
5::#8#89; /j../
))*
 8j!6!679 NP K Pyy55 ,-6
MF

3   !!$$--&&* {{zz{{{{{{}}}}}}

."# E
EE E C	E
  E &EP23 C	3p  	(// 4(#40j  $!%[|7tE 5**77TR) S)X 5--4H) I) @E 5( 499??=& >& ;@e K 5##>F ?F 
7?? $1h>    4::DI J0 4))UPTU  V F DJJT\\499emmTU V 4&doo&s+ 4<<T:B ;B2 4$$$?" @" DMM?# $ 4::C C 4::C C 499! ! 4::! ! 4::%%&% '% 4::! ! 4;;D9> :>2 5))tD
 E
 4>>t<# =# 4;;.  .b 4$$$?499$74<<T:1 ; 8 @1 4<<T:> ;> 4::48V 9V 4??=J >J$ 4##> ? 4''TB C
BJ '<<RVW/// / 	/
 / / / / X/d //T (,/// / 	/
 / / / %/ //d ,,44$!!! ! 	!
 ! ! !!H ..66D (,!!! ! 	!
 ! ! %! !!H ,,33*** * 	*
 * * **Z ..554 (,,,, , 	,
 , , %, ,,^ 488y: y:x 4==d;3E <3El 4%%4@6 A6 4((dC D 4;;D96 :6
 4::48 98 4((dC  D  4;;D9 : 4;;D9D :D4 4>>t< = 4??= > 488 $ T <3"<J/Nd	, 599%%11tL#+ M#+L 4&&DAG BG 4??= > 4>>##>' ?' 4<<  ! QB B/
 )):):; *499+>+>? )$***<*<= +DJJ,@,@A  dll  499J J 4::J J >44$OL PL
 >&&DAK BK >''TBJ CJ
 >--4H	 I	 >((dCRS  D6 >))tDLM	(09FI E8. 4$$++F
 "&ZZ
Z 	Z
 Z Z  Z Z GZz 4>>t<
 ,,, 	,
 , =,^2$CN d'' ( d&& ' d(( ) d** + dll ' d&&U 3 d!! " d,,5 9 d!! " d"" # doo  d!!)) * d++33 4 djj  dnn## $ d))11 2 d,,44 5 dkk  d$$5 1 d'')@ A doo} - d&&(: ; d!!#5 6 d..0B C d** + d== > d== > d22 3 d== > d33 4 d00- @ d00 1 d// 0 d// 0 d11 2 d11 2 d-- . d-- . d-- . d..0B C d00 1 d++] ; d"" # dii  dii  dmm  dii  dii  dkk  dnn  dmm  dll  doo  d  d-- . d   ! d'' ( d## $ dnn  d&& ' d"" # d$$ % dnn  d"" # d## $ d** + d  dnn  djj  d'' ( d// 0 d   d## $ d%% & d## $ d!! " djj  dmm  dll"" # dhhU + d((- 8 ell$$;; < ell$$77 8 ell$$AA B
 d!! " d** + d""$6 7 djj  d'' ( d;; < dnn  doo  djj  0088	
 99AA	
 ,,44	
 55==	
 ,,44	
 55==	
 44<<	
 ==EE	
 99AA	
 BBJJ	
 d++33_ E d,,44o F d//77 I d0088/ J d  
 499$7
 8
 4::"  ( 4"#+d**+E2 5:: ( 4&&DA B: 4%%4@@ A@F ELL$"4"456tDU 5 75p 5??#4 $4 5##$+ %+ 4++,+" -+"\ 4&&' ( 4//0%)59$ 1< 4>>t<K =K8 EKK,- 
 .$.> 0t/0B50IJ
1!45	 21 56
 4>>" $T$4  # 4%%&Td '> 4))*"4 +" 5%%--.A /A EJJ		*+: ,:
 4;;D9" :"J 4>>t< =8#F FR"1J 4::48
 9
 4%%4@/ A/ 4>>" # 4))* +;  4??= > >44$O P{| !1%%5!  0@,,44%0 ,
 4,,$G H, 4;;QUV	P W	P 1 1 4**E F 4<<T:9 ;9 !& 
  B 4==d;DH : <:" 4##>3 ?3 4$$$?6 @6 4&&DAP BP 4''TBPT m Cmh , *, 	,
 ,^ 4**223> 4> 41199:J ;J 4**223RVI-I@OI 4I 41199:RVU-U@OU ;U 4**223 !% $ $S S 	S
 S 4S 41199: !% $ $	 	 		
 	 ;	7 599$$% &*\\(\6;\\~ 4''TB4 C4n :><,> CG@F >B 5<<RVW 5 X5< 	33@ $4  (($   4//TJ  K8 -=))11- ) 488dSL TL^8B&R2j  0%%5  
 4,,-B .BJ  0$$%  
 4++,N -Nb "2&&E" 
2 4--.C /CL 433;;<BF* =*Z 'OO  'OO 
 4??=  >* 4??=  >*wt  0$$%  
 4++F b GbJ  0$$%  
 4++F D GDN$.b( 499:t : : 4 (V4( DHHeii()T5  * 4==!u  "   .HH  'txxER -HH 
 488t,, -,^ F 4::2 2 4::48 9     488t, - DHH:.( /(@ EII;$/$ 0$( 	txx'7DD

 DIIuzz*d;$ <$  DHHeii(). . *. #4;;#6#67#DLL$8$89 ():):)B)BC "4;;#6#67"4;;#6#67 4;;  . 4<<  !. 4$$% &* 4;;D9 :& 4;;D9 :& 499. . . 488@ @
 48867 77 48867 77 ! %--  	!: ;*		*>%+@A*		*>%+@A.!$++.85;;? /!$++.85;;? HH$L !!1!1uM 499##>!re $2 ?$2N 499$$$?H @H 	#4::.&txx0!$)),"4::.$))$
*4<<
8'		2	DKK	(t4   *   *" !1!12'(?(?@  -  0
()A)AB  !1!12 4;; ' * *I*V*V 4:: & 488 $ 499 %   * **	 !**	  OO**	
 !**	 T\\
*
T\\
* ! $.. !' * ! $.. !' *""'8
 4>> "$))L 499  4<<uzz B ! $.. !# & 477%** = 477%** = 477%** =uzzB 477%** = 477%** = 499 % 499 % 499 % 4:: & 499 % 4:: & 4:: & 499 % 4:: & 4== ) 499 % 4;; ' 4:: & 4:: & 499 % 4>> * D DE$ % 
D4Ld5 
0 	 3+		

 5Mt5 
0 	 3+		


, .CT  0ct  4,,33Sd K-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4,,113 7 4,,33S 9 4,,44c : 4,,44c : 4,,33S 9 4,,113 7 4,,<<c B-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4--t 4 4.. 6 40055w ? 40077 A 40055w ? 40077 A 42277 A 422997 C 42277 A 422997 C 433Z @ 4--t 4 4--t 4, T..335E t00779K T..335E t00779K T..335E t00779K
 C   ""K 0 ))+= > ""K 0 !!: . **,? @ ""K 0 C   !!3 ' && 1 ""K 0 ""K 0 !!: . ""K 0 C   T "  (   $,,  , " $// "#5 6  $++ z * " $// "#6 7  $,,  ,  - !!4?? 3 t{{ + !!4?? 3  - 4++, - 4==$$% &  4??&&' ( 4>>" # (M'') 8LFD1(01$78 5==! " 4<< 7 !7 599++334 5
 599%%;;< = 599>>&&445K 6K 599>>7#uyy~~++334# 5# 599>>(()%) = *=@ K J ! " 12S 3S( 599))..DI/ J/ 599))44$O/ P/ 599))99tT/ U/
 5**77TRDH  S8 &DA  4G  B F 599??//778 9 599))66DQ+ R+& 3 2   >88dS2 T2l     ! ! +  * * , '  ' ' )   ,  + + -   %  $ $ &  r\   