
    Vh                     t,   d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZmZm	Z	 d dl
mZ d dlZd dlmZ d dlmZ d dlZd dlmZmZmZ d dlmZmZmZ d dlmZ d dlmZ d dl m!Z! d d	l"m#Z# d d
l$m%Z% d dlm&Z&m'Z'm(Z( d dl)m*Z*m+Z+ ddl,m-Z-m.Z. ddlm/Z/m0Z0m1Z1  ed      Z2 ed      Z3 ejh                  e5      Z6ejn                  jp                  Z8ejn                  jr                  Z9ejn                  jt                  Z:ejn                  jv                  Z;ejn                  jx                  Z< eg e8jz                  e8j|                  e8j~                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e#e8j                  e8j                  e8j                  e8j                  j                  e8j                  j                  e:j                  e;j                        Zgi  e       egZhe8j                  e8j                  e8j                  e8j                  j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  e8j                  j                  e8j                  e8j                  e8j                  e8j                  gZ{ eehe{       de|e	ej                  j                  ej                  j                  f      deee3e2f   gee3e2f   f   fdZ ee8j                  j                  g      dej                  deddfd       Z ee8j
                  j                  g      dej                  deddfd       Z ee8j                  j                  g      ddddej                  deej                  j                     deej                  j                     ddfd       Z ee8j                  g      e	 	 ddej                  deej                  j                     deej                  j                     dej                  fd              Z ee8j                  g      d e|e	ebej                  f      d!ej                  j                  d"edej                  fd#       Z ee8j                  g      dd$dej                  d%ebd&ej                  dej                  d'ej                  j                  dej                  fd(       Zq ee8j                  j                  g      d e|e	ebej                  f      d)e|eb   d"edej                  fd*       Z ee8j                  g      d+ej                  d,ej                  d-ej                  d.e|eb   d/e	ebe|eb   f   d0e	ebe|eb   f   d1e	ebe|eb   f   d2ed3e|eb   d4ebd5e|e   deej                  ej                  ej                  f   fd6       Z ee8j"                  j$                  g      ddej                  d7ebdej                  fd8       Z ee8j(                  g      ed9ej                  d:ej                  dej                  fd;              Z ee8j*                  g      e	 	 dd9ej                  d<ej                  d=ej                  d>ej                  j                  d'ej                  j                  dej                  fd?              Z ee8j,                  g      ed9ej                  d@ej                  dej                  fdA              Z ee8j.                  j                  g      	 ddBe|ej                     d%ebdej                  fdC       Z ee8j0                  g      dej                  dej                  fdD       Z ee8j2                  g      dd$dej                  dEej                  d'eej                  j                     dej                  fdF       Z ee8j4                  g      d9ej                  dej                  fdG       Z ee8j6                  e8j8                  g      d9ej                  dej                  fdH       Z ee8j:                  e9j:                  g      d9ej                  dIej                  dej                  fdJ       Z ee8j<                  e9j<                  g      d9ej                  dIej                  dej                  fdK       Z ee8j>                        	 	 dd9ej                  d%eeb   dMedej                  fdN       Z ee8j@                        	 	 dd9ej                  d%eeb   dMedej                  fdO       Z ee8jB                  g      d9ej                  d%ebdPebdQebdej                  f
dR       Z ee8jD                  j                  g      d9ej                  d e|e	ebej                  f      dej                  fdS       Z ee8jD                  jH                  g      d9ej                  dTejH                  dej                  fdU       Z	 ddej                  dVeejL                     dejL                  fdWZ ee8jP                        ddddXd9ej                  dTeejH                     dYeejR                     dVeejL                     d"edej                  fdZ       Z ee8jT                        ddddXd9ej                  dTeejH                     dYeejR                     dVeejL                     d"edej                  fd[       Z ee8jV                        ddddLdLejX                  d\d9ej                  d!e	ebef   dTeejH                     d]eej\                     dYeejR                     d^ed_edVejL                  dej                  fd`       Z ee8j^                  j                        ddddXd9ej                  daebdTeejH                     dYeejR                     dVeejL                     d"edej                  fdb       Z ee8j^                  j`                        ddddXd9ej                  dcebdaebdTeejH                     dYeejR                     dVeejL                     d"edej                  fdd       Z ee8jd                  j                        daebd e|e	ebej                  f      d"edej                  fde       Z ee:j                  j                        d,ej                  d-ej                  dfej                  dej                  fdg       Ze ee;j                  j                        d,ej                  dhej                  diej                  d-ej                  djej                  dkej                  dfej                  dlej                  dmej                  dnebdej                  fdo       Zf eejn                  jt                  jf                        dpej                  dej                  fdq       Z ee8jj                  g      e	 	 	 ddrej                  dsej                  dtebduebdvedej                  fdw              Z ee8jl                  jn                        	 dd9e|ej                     dxe|ej                     dye|ej                     dzede|ej                     f
d{       Z ee8jr                  jn                        	 dd9e|ej                     dxe|ej                     dye|ej                     dzede|ej                     f
d|       Z ee8jv                  jn                        d}e|ej                     d~e|ej                     d-ej                  j                  de|ej                     fd       Z ee8jv                  jz                        d}e|ej                     d~e|ej                     de|ej                  j                     de|ej                     fd       Ze8j~                  j                  j                  ej                  j                  j                         ee8j~                        d,ej                  d-ej                  dfej                  ej                     dej                  ej                     dej                  ej                     dedededeej                  ej                  ej                  f   fd              Z e j                  d      deeedef   f   fd       Zdeeedef   f   fdZ ee8j                        d9ej                  dej                  dej                  dej                  fd       Z ee<j                  j                        d,ej                  debdebdedTejH                  deej                  ej                  f   fd       Z ee8j                        	 dd9ej                  d&ej                  dej                  dedej                  f
d       Z ee8j                        	 dd9ej                  d&ej                  dej                  dedej                  f
d       Z ee8j                  j                        ed+ej                  dej                  d%ebdejH                  dej                  f
d              Zn ee8j                        ddd9ej                  d%ebd&ej                  dej                  dededej                  fd       Z ee8j                        	 	 	 	 ddej                  de|eb   d/ee	ebe|eb   f      d0e	ebe|eb   f   d1e	ebe|eb   f   dedeej                  ej                  f   fd       Z ee8j                        dej                  de|eb   deej                  ej                  f   fd       Z ee8j                  jn                        dLdLddddej                  d9ej                  j                  dededee   deej                     dej                  fd       Z ee8j                        	 	 	 	 dd9ej                  dej                  dedededeej                     deej                  ej                  f   fd       Zy)    N)AnyCallableOptionalTypeVarUnion)	ParamSpec)core_aten_decompositionsget_decompositionsremove_decompositions)_grid_sampler_2d
_index_addpw_cast_for_opmath)extra_random_decomps)counters)	is_fbcode)	out_dtype)pad_listlike)elementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDtype_to_dtype)definitely_trueguard_size_oblivious   )configinductor_prims)is_gpu,needs_fallback_due_to_atomic_add_limitationsuse_scatter_fallback_T_Popsreturnc                     t        |       r| gn| D ]!  }|t        v st        j                  d|        # t	        j
                  | t              S )Nzduplicate decomp: %s)callabledecompositionslogwarningdecompregister_decomposition)r!   ops     M/home/dcms/DCMS/lib/python3.12/site-packages/torch/_inductor/decomposition.pyr)   r)   }   sI      }se# 5KK.45 ((n==    tensormsgc                      y N r-   r.   s     r+   assert_async_msg_decompr3          
r,   c                      y r0   r1   r2   s     r+   "functional_assert_async_msg_decompr6      r4   r,   )minmaxsymbolr7   r8   c                     y r0   r1   )r9   r7   r8   s      r+   sym_constrain_range_for_sizer;      s     r,   xc                 R    || j                  |      } || j                  |      } | S r0   )	clamp_min	clamp_max)r<   r7   r8   s      r+   clampr@      s/     KK
KKHr,   size
fill_valuekwargsc                     |j                  d      }|.t        t        |            |d<   t        j                  | |fi |S t
        S )Ndtype)getr   typetorchfullNotImplemented)rA   rB   rC   rE   s       r+   rI   rI      sF     JJwE}'Z(89wzz$
5f55r,   alphadimindexrL   c                ~    t               s#| j                  t        j                  k(  rt        S t        | |||d|      S )NF)inplacerL   )r   rE   rH   bfloat16rJ   r   )r<   rM   rN   r-   rL   s        r+   	index_addrR      s3     ;177enn4!S%eLLr,   physical_layoutc                     dgt        |       z  }t        |      D ]
  \  }}|||<    t        j                  |D cg c]  }| |   	 c}fi |j	                  |      S c c}w Nr   )len	enumeraterH   emptypermute)rA   rS   rC   permpls         r+   empty_permutedr]      se     3T?D/* 1Q;;9AQ9DVDLLTRR9s   Agrad_outputinputweight
bias_sizesstridepaddingdilation
transposedoutput_paddinggroupsoutput_maskc                 .   |
d   rt        | j                  j                        st        S t        j                  | dgt        t        d| j                                     z         }t        j                  | |||||||||	|
d   |
d   dg      \  }}}|||fS )N   r   r   F)
r   devicerG   rJ   atensumlistrangerM   convolution_backward)r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   	grad_biasgrad_inpgrad_weight_s                  r+   rp   rp      s     q>(:(:(?(?!@qcDq+//:K1L,M&MNI#88	QQ/ Hk1 k9--r,   decimalsc                 H    d|z  }t         j                  | |z        d|z  z  S )Ng      $@g      ?)rl   round)r<   ru   ten_pow_decimalss      r+   	round_decry      s,    X~::a**+s5E/EFFr,   selfbatch2c                    t         j                  r| j                  j                  dk7  rkt	        | j
                  d   dk(        st	        |j
                  d   dk(        r5| j                  d      |j                  d      z  j                  d      }|S | j                  j                  dk(  rt	        | j                  d      dk(        ryt	        |j                  d      dk(        r\t        d   dxx   dz  cc<   t        j                  | j                  d      |j                  d      z  dd	      j                  d      S t        S )
Ncpur   rj   rM   inductordecompose_bmmTrM   keepdim)r   coordinate_descent_tuningrk   rG   r   shape	unsqueezerm   rA   r   rH   squeezerJ   )rz   r{   outs      r+   bmmr     s    ''DKK,<,<,E

1 237KLLOq 8
 >>"%(8(8(;;@@Q@GCJ{{5 		! 127KKKOq 8
 Z 1Q6199Q&.."44!Til r,   mat1mat2betac                    | j                   j                  dk(  r7t        |j                  d      dk(        rt        |j                  d      dk(        rgt        d   dxx   dz  cc<   t        j                  |j                  d      |j                  d      z  dd      j                  d      }||z  || z  z   S t        |j                  d      dk(        ryt        |j                  d      d	k        r\t        |j                  d      d	k        r?t        d   dxx   dz  cc<   |j                  |z  j                  dd      }||z  || z  z   S t        S )
Nr}   r   r   r~   r   decompose_addmmTr      )rk   rG   r   rA   r   rH   rm   r   r   r   TrJ   )rz   r   r   r   rL   r   s         r+   addmmr     s2    {{5 		! 127KIIbMQ8
 Z !23q83))Q$,,r"224il  3;,, 1!23		! 23		! 23Z !23q8366D=%%!T%:C3;,,r,   input2c           	         t         j                  r| j                  j                  dk7  rit	        | j
                  d   dk(        st	        |j
                  d   dk(        r3| j                  d      |j                  d      z  j                  d      S | j                  j                  dk(  rt	        | j                  d      dk(        rt	        | j                  d      dkD        rt	        |j                  d      dk(        r| j                  |j                  k(  rt        t        j                  |       t        j                  |      z   dk        rYt        d   d	xx   dz  cc<   t        j                  t        | j                  d            D cg c]  }| |d d f   |z   c}      S t	        | j                  d      dk(        ryt	        |j                  d      dk(        r\t        d   d	xx   dz  cc<   t        j                  | j!                  d      |j!                  d      z  dd
      j                  d      S t"        S c c}w )Nr}   r   r   rj   r   r~       r   decompose_mmTr   )r   r   rk   rG   r   r   r   rm   rA   rE   r   rH   numelr   catro   r   rJ   )rz   r   is      r+   mmr   5  s    ''DKK,<,<,E

1 237KLLOq 8
 NN1%(8(8(;;@@Q@GG{{5  2!!34$TYYq\A%56$V[[^q%89v||+T!2U[[5H!HR OPZ 0A509951;NOad1a4j61OPP		! 127KKKOq 8
 Z 0A5099Q&.."44!Til  Ps   I	tensorsc                    ddl m dt        j                  dt        ffd}t        t        ||             t              dk(  rd   j                         S dt              cxk  rt        |       k  r#n n t        j                  j                        S t              dkD  rt        fdD              rd   }t        |j                        }dk  rt        |j                        z   n|j                  t                      |j                        j                   | j#                  dz         j                         S t$        S )Nr   )r   r<   r"   c                     t        | j                        dk(  r | j                  d   dk(        ryt        | j                        k  r | j                     dk(        ryy)Nr   r   FT)rV   r   )r<   rM   r   s    r+   non_empty_tensorzcat.<locals>.non_empty_tensora  sW    " qww<1!5aggajAo!FQWW"6qwws|q7H"Ir,   r   c              3   ,   K   | ]  }|d    u   yw)r   Nr1   ).0tfiltered_tensorss     r+   	<genexpr>zcat.<locals>.<genexpr>  s!      )%&a  )s   )%torch.fx.experimental.symbolic_shapesr   rH   Tensorboolrn   filterrV   clonerl   r   defaultallr   insertr   expandflattenrJ   )r   rM   r   inpr   r   r   s    `   @@r+   r   r   Z  s/   
 KELL T 2 F#3W=>
!"((**	
S!"	1S\	1xx 0#66 q S )*:) & q!SYY&)AgcC		N"3S#./0(s}}S!((%088cAgFLLNN r,   c                 "   | j                         ret        j                  t        j                  | j                        t        d      t        j                  | j                  | j                              S t        | t        j                        \  }}t        j                  t        j                  || j                        }t        j                  | dk  |d      }t        j                  t        j                  |       t        d      |      S )Nnan)type_promotion_kindrE   rk   r   g        )
is_complexrH   whereisnanrealfloatatan2imagr   r   INT_TO_FLOATscalar_tensormathpirk   )r<   rt   rE   r   rets        r+   angler     s    ||~{{KKuu{{166166/J
 	
 "	;HHHAu 
		TWWE!((	CB
++a!eR
%C;;u{{1~uU|S99r,   yc                b   t        j                  |       xr | j                         }t        j                  |      xr |j                         }|r|st        S |}|||z  }t        j                  | j
                  |j
                        }dt         j                  dt         j                  fd} || j                  | j                  j
                              } ||j                  |j                  j
                              }	t        j                  ||	z   d      j                  |      }
|
S )Nr-   r"   c                     | j                   ^ }}|dz  dk7  rt        d      g ||dz  d}| j                  |      }|S )zNReshape tensor from [*initial_dims, last_dim] to *initial_dims, last_dim/2, 2]rj   r   zQThe size of the last dimension must be even to reshape it to [..., last_dim/2, 2])r   AssertionErrorview)r-   initial_dimslast_dim	new_shapereshaped_tensors        r+   reshape_tensor_complexz#add.<locals>.reshape_tensor_complex  s^     #),,x a<1 c 
 6l5HM515	 ++i0r,   )	start_dim)
rH   	is_tensorr   rJ   promote_typesrE   r   r   r   r   )r<   r   rL   x_is_complex_tensory_is_complex_tensorzcomplex_typer   
x_reshaped
z_reshapedresults              r+   addr     s      //!,?//!,?&9	AAI&&qww8Lu||  " (qvv||(<=J'qvv||(<=J]]:
2bAFF|TFMr,   c                 4    | j                         rJ d       | S )NzTODO: implement this)r   rz   s    r+   conj_physicalr     s     8"88 Kr,   c                     | S r0   r1   r   s    r+   liftr     s    Kr,   otherc                 b    t        j                  t        j                  |      || kD  z  | |      S r0   rH   r   r   rz   r   s     r+   fminr     '    ;;u{{5)UT\:D%HHr,   c                 b    t        j                  t        j                  |      || k  z  | |      S r0   r   r   s     r+   fmaxr     r   r,   Fr   c                 x    | j                   t        j                  k(  rt        j                  | ||      S t        S Nr   )rE   rH   r   anyrJ   rz   rM   r   s      r+   amaxr     -     zzUZZyy388r,   c                 x    | j                   t        j                  k(  rt        j                  | ||      S t        S r   )rE   rH   r   r   rJ   r   s      r+   aminr     r   r,   startlengthc                 N    t        j                  | |||      j                         S r0   )rH   narrowr   )rz   rM   r   r   s       r+   narrow_copyr     s"     <<c5&17799r,   c                 J    t         j                  | |      j                         S r0   )rl   r   r   )rz   rA   s     r+   view_copy_defaultr     s    
 99T4 &&((r,   rE   c                 @    | j                  |      j                         S r0   )tor   )rz   rE   s     r+   view_copy_dtyper     s    
 775>!!r,   memory_formatc                 X    |t         j                  u s|t        j                  |       S |S r0   )rH   preserve_formatutilssuggest_memory_format)r-   r   s     r+   get_like_layoutr     s.    
 ---1F**622r,   )rE   rk   r   rk   c                    t        j                  g | j                         f|xs | j                  |xs | j                  d|j                  t        | |            S Nr   r   )rH   randrA   rE   rk   r   r   rz   rE   rk   r   rC   s        r+   	rand_liker     s^     ::$))+!tzz$ 	
 	bt];b<=r,   c                    t        j                  g | j                         f|xs | j                  |xs | j                  d|j                  t        | |            S r   )rH   randnrA   rE   rk   r   r   r   s        r+   
randn_liker  /  s^     ;;$))+!tzz$ 	
 	bt];b<=r,   )rE   layoutrk   
pin_memoryrequires_gradr   r  r  r  c                    t        j                  g | j                         ||xs | j                  |xs | j                  |xs | j
                  |      j                  t        | |            S )N)rE   r  rk   r  r   )rH   rI   rA   rE   r  rk   r   r   )rz   rB   rE   r  rk   r  r  r   s           r+   	full_liker  @  sa     ::$))+!tzz$$# 	bt];b<=r,   highc                    t        j                  j                  d|g | j                         f|xs | j                  |xs | j
                  d|j                  t        | |            S )Nr   r   r   rl   randintlowrA   rE   rk   r   r   )rz   r  rE   rk   r   rC   s         r+   randint_liker  V  sl     <<	$))+ !tzz$  	bt];b<=r,   r  c                    t        j                  j                  ||g | j                         f|xs | j                  |xs | j
                  d|j                  t        | |            S r   r
  )rz   r  r  rE   rk   r   rC   s          r+   randint_like_lowr  j  sl     <<$))+ !tzz$  	bt];b<=r,   c                 F    t        j                  j                  d| |fi |S rU   )rl   r  r  )r  rA   rC   s      r+   r  r    s"     <<AtT4V44r,   biasc                     t         j                  j                  j                  |      }t         j                  j                  j	                  | |||j                         d         S rU   )rH   r!   
_quantized$wrapped_fbgemm_pack_gemm_matrix_fp16!wrapped_fbgemm_linear_fp16_weightrA   )r_   r`   r  packed_weights       r+   #linear_dynamic_fp16_unpacked_weightr    sO     II((MMfUM99AA}dFKKM!$4 r,   input_scaleinput_zero_pointweight_scaleweight_zero_point	out_scaleout_zero_pointout_channelc
           	          t         j                  j                  j                  ||||      }
t         j                  j                  j	                  | |||
|||	      S r0   )rH   r!   r  _wrapped_linear_prepack#_wrapped_quantized_linear_prepacked)r_   r  r  r`   r  r  r  r  r  r  r  s              r+   wrapped_quantized_linearr"    s\     II((@@/M 99CC r,   packedc                     dt         j                  dt         j                  fd} || dddf         } || ddd f         }| dd df   j                  t         j                        |z  |z   S )Nu8r"   c                      fddD        \  }}}}t         j                  dk(  r4||dz  z   |dz  z   |dz  z   j                  t        j                        d   S |dz  |dz  z   |dz  z   |z   j                  t        j                        d   S )Nc              3   f   K   | ](  }d |f   j                  t        j                         * yw.N)r   rH   int32)r   nr%  s     r+   r   zPq_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32.<locals>.<genexpr>  s&     GQbajmmEKK0Gs   .1)r   r   rj      little   r      r(  )sys	byteorderr   rH   float32)r%  r<   r   r   ws   `    r+   bitcast_u8_to_f32z=q_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32  s    G,G
1a==H$aLAG,R8>>u}}MiXX"Wb)Q!V4q8>>u}}MiXXr,   .i)rH   r   r   r1  )r#  r3  scalesoffsetss       r+   "q_embedding_bag_byte_unpack_decompr7    sz    Yell Yu|| Y vc2b5j12FsBCx 01G#ss(u}}-6@@r,   agridinterpolation_modepadding_modealign_cornersc                     | j                   t        j                   d      k(  xr' |dk(  xr  | j                  t        j                         }t	        | |||||      }|S )Nr}   r   r   )r9  r:  r;  r<  _expand_grid)rk   rH   is_contiguouscontiguous_formatdecomp_grid_sampler_2d)r8  r9  r:  r;  r<  r>  outputs          r+   grid_sampler_2drC    sm     	
ELL'' 	C!#	COO%*A*AOBL $	-!#!F Mr,   left_tensorsright_tensorsscalarc                     t         j                  j                  | t         j                  j                  ||      |      S NrK   )rl   _foreach_addList_foreach_mulrz   rD  rE  rF  s       r+   _foreach_addcmul_scalarrM    <     !!d$$\=A "  r,   c                     t         j                  j                  | t         j                  j                  ||      |      S rH  )rl   rI  rJ  _foreach_divrL  s       r+   _foreach_addcdiv_scalarrQ    rN  r,   start_tensorsend_tensorsc           	          t         j                  j                  | t         j                  j	                  t         j
                  j                  ||       |            S r0   )rl   rI  rJ  rK  Scalar_foreach_sub)rR  rS  r`   s      r+   _foreach_lerp_scalarrW    sL     !!  "";>	
 r,   scalarsc           	          t         j                  j                  | t         j                  j	                  t         j
                  j                  ||       |            S r0   )rl   rI  rJ  rK  
ScalarListrV  )rR  rS  rX  s      r+   _foreach_lerp_scalarlistr[    sL     !!$$"";>	
 r,   running_meanrunning_vartrainingexponential_average_factorepsilonc           
          t         j                  | |||||||      \  }}	}
|r||	|
fS ||j                  d      |j                  d      fS )Nr   )rl   native_batch_norm	new_zeros)r_   r`   r  r\  r]  r^  r_  r`  r8  bcs              r+   miopen_batch_normrg    sl     $$"	GAq! 1ay	 r,   .c                      i t         t        S r0   )r%   r   r1   r,   r+   fast_random_decompsri  0  s    5n5 455r,   c                  B    t         j                  rt        S t               S )z"decomps can change based on config)r   fallback_randomr%   ri  r1   r,   r+   select_decomp_tablerl  7  s      r,   masksourcec                    ddl m}m}  || j                  |j                        rt
        j                  | |g      \  } }|j                  d      j                  d      dz
  }d | ||fD        \  }}}t
        j                  |||gd      }	t        j                  ||	|      j                  | j                        S t        S )Nr   )BackendFeaturehas_backend_featurer~   r   c              3   <   K   | ]  }|j                           y wr0   )r   )r   r<   s     r+   r   z!masked_scatter.<locals>.<genexpr>K  s     ,WQQYY[,Ws   )codegen.commonrp  rq  rk   MASKED_SCATTER_WITH_INDEXrl   broadcast_tensorsreshapecumsum_unsafe_masked_indexrH   r   r   r   rJ   )
rz   rm  rn  rp  rq  
source_idx	self_flat	mask_flatsource_flatr   s
             r+   masked_scatterr}  >  s     D4;;(P(PQ ++T4L9
d\\"%,,Q/!3
,W4vBV,W)	9k**;	J<QRS{{9fi8==djjIIr,   	quant_min	quant_maxepsc                    t        j                  |       \  }}||z
  t        ||z
        z  }t        j                  |t        j                  |g            }|t        j
                  ||z        j                  t         j                        z
  }t        j                  |||      }|j                  t         j                        |j                  t         j                        fS r0   )rH   aminmaxr   r8   r   rw   r   intr@   float64int64)	r_   r~  r  r  rE   min_valmax_valscale
zero_points	            r+   choose_qparams_tensorr  Q  s     }}U+GWw%	I(=">>EIIeU\\3%01EU[[59<<UYYGGJZI>J88EMM"JMM%++$>>>r,   
accumulatec                     | j                         }t        j                  ||g|j                  |j                        |      }|j                  | j                        S r0   )r   rH   	index_putrv  r   )rz   rN   rn  r  	flatteneds        r+   putr  a  sL     IE7FNN5;;7I TZZ((r,   c                 V    t         j                  | |||      }| j                  |      S )N)r  )rl   r  copy_)rz   rN   rn  r  r   s        r+   put_r  o  s(     ((4:(
>C::c?r,   rB  input_dtypec                     | |z  }t        j                  ||d      }t        j                  | ||      }| j                  |k7  r|j                  |      }|j                         S )NTr   )rH   rm   r   fmarE   r   
contiguous)r^   rB  rM   r  new_grad_outputsum_new_grad
grad_inputs          r+   _softmax_backward_datar  z  sc     "F*O99_#tDL##VG\?KJ K']];/
  ""r,   Tinclude_selfsrcreduction_typer  c                   |dk(  rt        | j                        s| j                  j                  xs | j                  j                  }t	        j
                  |      }|r)| }t	        j
                  |       j                  |||      }	nN| j                  ||d      }t	        j                  |       j                  |||      }	|	j                  |	dk  d      }	|j                  |||      }|r||	z  S ||	z  S t        t        j                  j                  || j                  |j                  |j                  j                  d      rt         S | j"                  |dz   d  j%                         | j"                  d | j%                         z  }
|j%                         g| j"                  |dz   d  | j"                  d | }g t'        | j(                  |z
  | j(                        dt'        d| j(                  |z
        }|j+                  t        j,                        j/                  |
      j1                  |      j3                  |      }| j5                  |||||      S )Nmeanr   r   Tr  )r   rE   is_floating_pointr   rH   	ones_likerR   
index_fill
zeros_likemasked_fillr   rl   scatter_reduce_twork   rG   rJ   r   r   ro   ndimr   r  repeat_interleaverv  rY   scatter_reduce)rz   rM   rN   r  r  r  true_divisiononesr   countsrepeatsindex_shaperZ   scatter_indexs                 r+   index_reducer    s    (T

) 

44M

8M8Ms#C__T*44S%FF//#ua0C%%d+55c5$GF''
A6FmmC,,sV|?#-?  

		

 jjq#))+djj#.>.D.D.FFG;;=L4::cAgi#8L4::ds;KLKNU499s?DII.NNE!TYY_4MND		7	#				  !   r,   kernel_size	ceil_modec                    |dk(  rddg}|dk(  rddg}|s|}t        |d      }t        |d      }t        |d      }t        |d      }|d   |d   z  }t        j                  j                  j	                  ||      s0|t        j
                  t        j                        j                  kD  rt        S t        j                  | |||||      \  }}t        j                  ||d   | j                  d      ||      }	||	fS )Nr   r   rj   r~   )r   rH   	_inductorlowering'should_fallback_max_pool2d_with_indicesiinfoint8r8   rJ   prims#_low_memory_max_pool2d_with_offsets)_low_memory_max_pool2d_offsets_to_indicesrA   )
r<   r  rb   rc   rd   r  window_sizevalsr6  indicess
             r+   max_pool2d_with_indicesr    s    1}q6!|a&{A.KHa(H7A&G&!$Fa.;q>1K 	  HH	
 UZZ0444==	MD' ==A	r
G =r,   output_sizec                     | j                   ^ }}}|\  }}|dk(  s|dk(  r:g |||}| j                  |      | j                  |t        j                        fS ||z  dk(  r(||z  dk(  r ||z  ||z  g}t        j                  | |      S t        S )Nr   )rE   )r   	new_emptyrH   r  rl   r  rJ   )	r<   r  batchh_inw_inh_outw_outo_sizer  s	            r+   adaptive_max_pool2dr    s     UD$LE5zUaZ'5'%''{{6"AKKekkK$JJJe|qTE\Q.u}dem4++A{;;r,   	out_int32rightsidesortersorted_sequencer  r  r  r  c                ~    t         j                  | t        j                  |g| j                        ||||      d   S )N)rk   r  r   )rl   searchsortedrH   r-   rk   )r  rz   r  r  r  r  s         r+   searchsorted_scalarr  
  sK     dVO$:$:;   	 	r,   noiselowerupper	generatorc                    |rS| dk  }t         j                  | |||      }t        j                  || |z  |       }t        j                  ||d      }	||	fS ||z   dz  }
t         j	                  | |
      t        j
                         fS )Nr   )r  r   rj   )rl   uniformrH   r   
leaky_relur   )rz   r  r  r  r^  r  not_positiverrB  	noise_outnegative_slopes              r+   rrelu_with_noise_functionalr    s     qyLLueyLA\4!8T:KKa3	y  %-1,t^4ellnDDr,   )NNrb  )r   r   )NFr0   )r   r   F)r   )F)Nr   r   F)g      ?gUUUUUU?FN)	functoolsloggingr   r/  typingr   r   r   r   r   typing_extensionsr   rH   torch._decomp_decompr(   torch._prims_common_prims_commonr   $torch.ao.quantization.fx._decomposedr	   r
   r   torch._decomp.decompositionsr   rA  r   r   $torch._decomp.decompositions_for_rngr   torch._dynamo.utilsr   torch._environmentr   !torch._higher_order_ops.out_dtyper   torch._inductor.utilsr   r   r   r   r   r   r    r   r   r   r   r   r   r    	getLogger__name__r&   r!   rl   r  	quantizedr  quantized_decomposed_adaptive_avg_pool2d_backwardindex_selectaddmvarangebitwise_and_bitwise_or_
clamp_min_dist
empty_likeflipgeluhardtanhlcmr  linalg_vector_norm_log_softmax max_pool2d_with_indices_backward_native_batch_norm_legit#_native_batch_norm_legit_functional$_native_batch_norm_legit_no_training_batch_norm_with_update"_batch_norm_with_update_functional_batch_norm_no_updatebatch_norm_backwardrc  native_group_normnative_layer_normnll_loss2d_backwardpermute_copyrrelu_with_noise_backward_softmaxsin_sqrt__to_copytril_indicestriu_indicesunbind_copyr  upsample_bilinear2dvecr  r"  inductor_decompositionsr%   _unsafe_indexrx  #_unsafe_masked_index_put_accumulate+_scaled_dot_product_flash_attention_for_cpur   r  r?   r>   rR   gluselect_scatterslice_scattersplitr   r   rm   unbindbaddbmmdecomps_to_excludern   _opsOperatorBaseOpOverloadPacketr)   _assert_asyncr.   strr3   _functional_assert_asyncr6   r;   SymInttypesNumberr@   rI   r]   rp   r   tuplerw   ru   ry   r   r   r   r   r   r   r   r   detach_r   r   r   r   r   	view_copyr   rE   r   r   r   r   rk   r  r  r   r   r  r  	low_dtyper  r  embedding_bag_byte_unpackr7  rC  _foreach_addcmulrU  rM  _foreach_addcdivrQ  _foreach_lerprW  rZ  r[  rg  py_impl_CDispatchKeyAutograd	lru_cachedictri  rl  r}  choose_qparamsr-   r  r  r  r  r  r  r  r  r  	Generatorr1   r,   r+   <module>r=     ss      
  : : '   # + 
 
 F ( ( 7 . 
 X $  T]t_g!yy~~		II	YY!!
yy55 ,***** 	

* 		*
 	* 	* 	* 			* 	* 			* 			* 	* 	* 	* 	*  	!*" 	--#*$ 	%%%*& 	00'*( 	11)** 	$$+*, 	//-*. 	""/*0 	  1*2 	3*4 	5*6 	7*8 	  9*: 	;*< 	&&=*> 	?*@ 			A*B 	

C*D 	E*F 	G*H 	I*J 	K*L 	M*N 	  $$O*P 	55Q*R 	++S*, Z K,.J2IJ
 	,,44<<NNNNNNHHJJLLHHKKLL! & n&8 9>	eEJJ++UZZ-H-HHI	J>xB (2r6"223> ++//01ELL s t  2
 66::;<u|| # $  = ::BBCD )-(,	LL 
%++$$	% 
%++$$	%	
 
 E % )-(,	||		%++$$	%	 
%++$$	%	 \\		  &	 $	
uS%,,&'
(	""	 	 \\		 %	 () !"M||M	M <<M LL	M ;;M \\M *M* ,,4456S
uS%,,&'
(S#YS S \\	S 7S 2234..<<. LL. S		.
 #tCy.!. 3S	>". CcN#. . I. . d. 5<<u||34. 5.@ ,,-.G G GU\\ G /G
 
#
,,LL \\  $* %
  ! !
,,
,, ,, ++

	
 ;; \\  &6 	"
,,LL \\  #F ))*+ 2%,,2	2 \\2 ,2j %:U\\ :ell : &:& 
#
 +/	&||&||& EKK&&'	&
 \\& $&R ++,-   .
 DLL12u||   3 EJJ/0Iu|| IELL IU\\ I 1I EJJ/0Iu|| IELL IU\\ I 1I 		" 
,,	#  \\	 # 		" 
,,	#  \\	 # ))*+:
,,:	: : 	:
 \\: ,: //01)
,,)
uS%,,&'
() \\) 2) --./"
,,";;" \\" 0" 48LLE//0  ' $(%)37=
,,= EKK = U\\"	=
 E//0= = \\= (=  ( $(%)37=
,,= EKK = U\\"	=
 E//0= = \\= )=  '
 $(%)%)).)>)>=
,,=c5j!= EKK 	=
 U\\"= U\\"= = = &&= \\= (=* ))112
 $(%)37=
,,=
= EKK 	=
 U\\"= E//0= = \\= 3=& ))334 $(%)37=
,,=	= =
 EKK = U\\"= E//0= = \\= 5=( ,,-5
5
uS%,,&'
(5 5 \\	5 .5 	EEMMN<<LL ,, \\	 O 
;;CCD<< ll LL	
 ,, || ,, || LL  \\ E4 		++EEF
Au|| 
A 
A G
A --./  ||
,,  	
  \\  0: --445
 	
u||
u||$ % 	
 
%,, 6 --445
 	
u||
u||$ % 	
 
%,, 6 **112
%
ell#
 KK
 
%,,	
 3
 **556
%
ell#
 %++$$%
 
%,,	
 7
 ''(<(<(E(EF../<<LL //%,,
' //%,,/	
 .  !&  5<<u||34 0 G< T6T#xS'9"9: 6 6!T#xS'9"9: ! ++,
,,
,, LL \\	 -$ ,;;BBC?<<?? ? 
	?
 ;;? 5<<%&? D? !
 	
)
,,
)<<
) LL
) 	
)
 \\
) "
) 		"
 	
,,<< LL 	
 \\ # 33;;<##LL# 
# 	#
 \\#  =#( ))* 1
,,1	1 <<1 
	1
 1 1 \\1 +1h 445 /3%&&'/||/c/ U3S	>*+/ 3S	>"	/
 CcN#/ / 5<<%&/ 6/d 001||"&s)
5<<%& 2" ))001
 %)	\\	
++

	 		
 	 3-	 U\\"	 \\	 2	& 889 %+/E
,,E<<E E 	E
 E (E 5<<%&E :Er,   