
    Vh                        d dl Z d dlmZ d dlZd dlmc mZ d dlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZmZ ddlmZmZmZ ddlmZm Z m!Z! ddl"m#Z#m$Z$ 	 	 	 	 ddede%e   de%e   fdZ&de&_'        d Z(y)    N)Optional)mm_args   )ir)CppGemmTemplate)CppGroupedGemmTemplatecreate_epilogue_with_attr)	TensorBox)addadd_needs_realized_inputsatenpermuteregister_loweringto_dtypeview)autotune_select_algorithmChoiceCallerExternKernelChoice)use_aten_gemm_kernelsuse_cpp_gemm_templateuse_max_autotune)opsVxwbc           
         | j                         }t        |      dkD  rt        | d|d   g      } t        |      }t               sJ |D 	cg c]%  }	|	|	nt        j
                  j                  |	      ' }}	g }
t        | t        |d   ddg      |      ^ }}} }t        |D 	cg c]  }	|	d u c}	dd t        |      D ci c]  }||  c}      }| g|}|j                  |D 	cg c]  }	|	|		 c}	       t        j                  |
||fi | t        |
      dk7  sJ t        d|
||      }|j                  j                  }t        |      D cg c]   }t	        j                   ||t"        |fg      " }}t	        j$                  |d   j'                         	      |_        ||_        t        |      D cg c]$  }t        j,                  j/                  ||         & }}t        |      dkD  r>t        |      D ]0  }t        ||   g |d d ||   j                         d         ||<   2 |S c c}	w c c}	w c c}w c c}	w c c}w c c}w )
N   r   r   layoutT)has_biastrans_wepilogue_creatoract_mappinggrouped_gemm)device)get_sizelenr   r   r   ExternKernelrealize_inputr   r   dictrangeextendr   add_choicesr   dataMultiOutputlistMultiOutputLayout
get_devicer"   outputsr   create)r   r   r   attrscalars	algorithmr"   x_sizenum_gemmbiaschoices_numkwargsinput_nodesresulttemplate_bufgemm_idxreturn_bufsreturn_tensorss                       P/home/dcms/DCMS/lib/python3.12/site-packages/torch/_inductor/mkldnn_lowerings.pygrouped_gemm_loweringrI       st    ZZ\F
6{QR$%1vHSTU42??#@#@#F	FUAU"$Gq'!A$A"7GQ1/01t$d"1',X7S!V7	F 'q'K?d.>?@&& 	 w<1&	F ;;##L h 	v|tX.>-?@K  ..k!n6O6O6QRL&LCH?7?K12N  6{Qh 	H'+x(G&"+G~h7@@B2FG(N8$	
 [ 	V 2 8 @"s*   *H2$H7 
H<&I.I%I9)ITc            !         t         j                  j                  rYddlm t        t         j                  j                  j                  ddj                  j                        t        t         j                  j                  j                  j                  ddj                  j                        t        t         j                  j                  j                  ddj                  j                        t        t         j                  j                  j                  j                  ddj                   j                        t         j                  j                  j"                  t         j                  j                  j$                  t         j                  j                  j&                  t         j                  j                  j                  t(        j*                  j,                  t         j                  j                  j.                  g} t1        t         j                  j                  j"                        dt2        dt2        d	t2        ffd
       }t1        t         j                  j                  j"                  j                        dt2        dt2        dt2        d	t2        ffd       }t1        t         j                  j                  j$                  j                        dt2        dt2        dt2        d	t2        ffd       }t1        t         j                  j                  j                        	 d3dt2        dt2        dt2        ffd       }t1        t         j                  j                  j                  j                        	 d3dt2        dt2        dt2        dt2        ffd       }t1        t         j                  j                  j&                        dt2        dt2        d	t2        ffd       }t1        t(        j*                  j,                        dt2        dt2        dt2        dt2        dt2        dt2        dt2        dt4        dt6        t8           dt8        dt8        dt8        dt4        d t4        d!t4        d"t4        f fd#       }t1        t         j                  j                  j.                  d $      dt2        d%t2        d&t2        d't2        d	t2        f
fd(       }t1        t         j                  j                  j.                  j                  d $      t1        t         j                  j                  j.                  j:                  d $      dt2        d%t2        d&t2        d't2        d)t2        d	t2        ffd*              }	t1        t         j                  j                  j                  d $      	 d3dt2        d%t2        d&t2        d't2        d	t2        f
fd+       }
t1        t         j                  j                  j                  j                  d $      t1        t         j                  j                  j                  j:                  d $      	 d3dt2        d%t2        d&t2        d't2        d,t2        d	t2        ffd-              }t         j                  j<                  rt        t         j                  j>                  j@                  d.djB                  j                        | jE                  t         j                  j>                  j@                         t1        t         j                  j>                  j@                        d d/dt2        d0t2        d1t2        dtF        t2           ffd2       }tI        |        y y )4Nr   )	mkldnn_irzmkldnn::_linear_pointwiseF)has_out_variantkernel_creatorzonednn::qlinear_pointwiser   weightr=   c
                 r    t        j                  
j                  j                  | |||||||||	
            S N)r   r7   ConvolutionUnary)r   rN   r=   paddingstridedilationgroupsr8   r9   r:   rK   s             rH   convolution_unaryz5register_onednn_fusion_ops.<locals>.convolution_unary   sJ     ##**11     otherc                 x    t        j                  j                  j                  | |||||||||	|
||            S rP   )r   r7   ConvolutionBinaryr   rX   rN   r=   rR   rS   rT   rU   binary_attrbinary_alpha
unary_attrunary_scalarsunary_algorithmrK   s                rH   convolution_binaryz6register_onednn_fusion_ops.<locals>.convolution_binary   sS      ##++22 !# rW   c                 x    t        j                  j                  j                  | |||||||||	|
||            S rP   )r   r7   ConvolutionBinaryInplacer[   s                rH   convolution_binary_inplacez>register_onednn_fusion_ops.<locals>.convolution_binary_inplace   sS      ##2299 !# rW   r   r   c                 ^   | j                         }t        |      dkD  rt        | d|d   g      } |t        j                  j                  |      }g }t               rvt        |ddg      }	t        | |	|      ^ }
}} }	t        || |	      rHfd}t        |d uddk(  rd n|	      }|g d
|d<   t        j                  |||| |gn| ||gfi | t        |      dk(  s
t               rAt              }|d |d<   |j                   j                  || |gn| ||g|fi |       |j!                         t"        j$                  j&                  v sJ dd i}t)        d||| |gn| ||g||      }t        |      dkD  r%t        |g |d d |j                         d         }|S )Nr   r    r   r   r!   c                 "    t        |       S )Nr9   r:   r	   )bufr:   r8   r9   s    rH   r%   zJregister_onednn_fusion_ops.<locals>.linear_unary.<locals>.epilogue_creator   s    8w)  rW   Tnoner#   r$   r%   )r   r   r   input_indices)r8   r9   r:   Bc                 X    t         j                  j                  | j                            S rP   r   graph	constantsget_namer   s    rH   <lambda>zBregister_onednn_fusion_ops.<locals>.linear_unary.<locals>.<lambda>      QWW..qzz|< rW   linear_unaryinput_gen_fnsr)   r*   r   r   r+   r,   r   r   r   r   r-   r   r0   r   appendbindrq   r   ro   rp   r   )r   r   r   r8   r9   r:   r"   r;   r>   transposed_wr?   r%   rA   rw   rC   aten_mkldnn_linear_unarys      ```         rH   ru   z0register_onednn_fusion_ops.<locals>.linear_unary   s    ZZ\F6{QR,-}OO11!4*,G!&q1a&1.5af.U+FA|(LA
 "!"$ $15EUF
 }2;/#//"#)A!Q !	 7|q $9$;4IN9"&F3K1,11"#)A!Q ! ::<177#4#4444<M /)A!Q+F 6{Qf&Ks&KV__5Fr5J&KLMrW   yc                    | j                         }t        |      dkD  rt        | d|d   g      } j                         }t        |      dkD  rt        d|d   g      |t        j                  j                  |      }g }t               rvt        |ddg      }	t        | |	|      ^ }
}} }	t        || |	      rFfd}t        |d ud|      }|g d	ng d
|d<   t        j                  |||| |gn| ||gfi | t        |      dk(  s
t               rAt              }|d |d<   |j                   j                  || |gn| ||g|fi |       |j!                         t"        j$                  j&                  v sJ dd i}t)        d||| |gn| ||g||      }t        |      dkD  r%t        |g |d d |j                         d         }|S )Nr   r    r   r   r!   c                      t        |       S )N)rX   r	   )rh   r8   r}   s    rH   r%   zKregister_onednn_fusion_ops.<locals>.linear_binary.<locals>.epilogue_creator?  s    8d!LLrW   Trj   )r   r   r   )   r   r   r   rk   )r8   rl   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zCregister_onednn_fusion_ops.<locals>.linear_binary.<locals>.<lambda>[  rt   rW   linear_binaryrv   rx   )r   r}   r   r   r8   r"   r;   y_sizer>   r{   r?   r%   rA   rw   rC   aten_mkldnn_linear_binarys    `  `          rH   r   z1register_onednn_fusion_ops.<locals>.linear_binary*  s$    ZZ\F6{QR,-ZZ\F6{QR,-}OO11!4*,G!&q1a&118|Qv2.FA|Q )LAM "!"$ $)9F
 <=9i,F?+#//%&YAq	Q1aL !	 7|q $9$;49"&F3K2-22%&YAq	Q1aL ! ::<177#4#4444<M /YAq	Q1aL+F 6{Qf&Ks&KV__5Fr5J&KLMrW   c                 t    t        j                  j                  j                  | |||||||||	|
            S rP   )r   r7   ConvolutionTransposeUnary)r   rN   r=   rR   output_paddingrS   rT   rU   r8   r9   r:   rK   s              rH   convolution_transpose_unaryz?register_onednn_fusion_ops.<locals>.convolution_transpose_unaryh  sM     ##33::" rW   w0w1w2w3hxcxreversebatch_sizesmodehidden_size
num_layers
has_biasesbidirectionalbatch_firsttrainc                     t        j                  t        j                  j                  j                  | |||||||||	|
|||||            S rP   )pytreetree_mapr   r7   MkldnnRnnLayer)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rK   s                   rH   mkldnn_rnn_layerz4register_onednn_fusion_ops.<locals>.mkldnn_rnn_layer  sc    & ??  ((//!! rW   )type_promotion_kindpacked_weightw_scalew_zpc                    t        |      t        k(  sJ t        j                  j	                  t        j                  |t
        j                        d      }t        |      t        k(  sJ t        j                  j	                  t        j                  |t
        j                        d      }t        j                  j                  j                  | |||||||||	|
||||||            S )Ndtypex_scalenamex_zp)typefloatr   ro   add_tensor_constanttorchtensorfloat32intint32r   r7   QConvPointWisePT2E)r   r   r   r   r   r   r=   rS   rR   rT   rU   o_inv_scaleo_zero_pointoutput_dtyper8   r9   r:   rK   s                    rH   qconvolution_unaryz6register_onednn_fusion_ops.<locals>.qconvolution_unary  s    * =E)))gg11WEMM: 2 G :$$$77..T5F / D ##,,33!  # rW   accumc                    t        |      t        k(  sJ t        j                  j	                  t        j                  |t
        j                        d      }t        |      t        k(  sJ t        j                  j	                  t        j                  |t
        j                        d      }|dk(  rq|t
        j                  t
        j                  fv rO|j                         t
        j                  t
        j                  fv r|j                         |k7  rt        ||      }t        j                  j                  j                  | |||||||||	|
|||||||||||            S )Nr   r   r   r   sum)r   r   r   ro   r   r   r   r   r   r   bfloat16	get_dtyper   r   r7   QConvPointWiseBinaryPT2E)r   r   r   r   r   r   r   r=   rS   rR   rT   rU   r   r   r   accum_scaleaccum_zpr\   alphar^   r_   unary_algorithmmrK   s                         rH   qconvolution_binaryz7register_onednn_fusion_ops.<locals>.qconvolution_binary  sF   > =E)))gg11WEMM: 2 G :$$$77..T5F / D
 u$ U]]ENN$CCOO%%--)HHOO%5 !5##2299!  !$- rW   c                   	
 |j                         t        j                  u sJ d       | j                         }t	        |      dkD  rt        | d|d   g      } t        t        j                        sYt              t        k(  sJ t        j                  j                  t        j                  t        j                        d      n^j!                          t#        d j                         D              rt        g       t	        j                               dv sJ d	       Dt        j                  j                  t        j                  d
t        j$                        d      t        t        j                        sYt              t&        k(  sJ t        j                  j                  t        j                  t        j$                        d      nj!                          j)                         dk(  sJ d       |Dt        j                  j                  t        j                  d
t        j$                        d      }j!                          |j!                          |j                         t        j$                  k7  rt        t        j*                  j-                  |      t        j.                        rt        j                  j0                  |j3                            j5                  t        j$                        }t        j                  j                  t        j                  |t        j$                        |j3                               }d nj                         g }t7               rt9        | ||	      ^ }}} }t        t        j*                  j-                  |      t        j.                        rt        j:                  t        j<                  t        j                  j0                  |j3                                  t        j                  j0                  |j3                                  r%t?        || |      rt        j                  j0                  |j3                            jA                         }t        jB                  |j5                  t        j                        d
      }t        j                  j                  ||j3                         dz         
	fd}| j                         t        jD                  t        j                  fv sJ tG        jH                  ||| ||gn| ||gd u|g dng d       t	        |      d
k(  s
tK               rLtM        	
      }d |d<   |jO                   jP                  | ||fn| ||f|fi |       |j3                         t        j                  j0                  v sJ d d d d d}t        t        j*                  j-                        t        j.                        rd |d<   t        t        j*                  j-                        t        j.                        rd |d<   tS        d|| ||gn| ||g||       }t	        |      dkD  r%t        |g |d d |j                         d         }|S )!Nz2Only int8 weights are supported by oneDNN qlinear.r   r    r   r   r   c              3   &   K   | ]	  }|d k(    ywr   N .0dims     rH   	<genexpr>zDregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.<genexpr>P       >Csax>   r   r   x_scale must be 0D or 1Dr   r   r   z(x_zp is incompatible with oneDNN qlinearr   r"   	out_dtyper   _BMatrixCompensc                 $  	
 t         j                  t         j                  t         j                  t         j                  fv sJ | j                         j                         j                         j                         
j                         d j                         
fd}t        j                  | j                         t         j                  || j                               }dk7  rt        |      }t         j                  k(  rM|j                         fd}t        j                  |j                         ||j                               }|S t         j                  t         j                  fv rzddlm |j                         		fd}t        j                  |j                         t        j                  |t!              t#              	      |j                               }|S )
Nc                     |       }t        j                  |t        j                        }| d   f} d      } d      } |      } |      }t        j                  t        j                  ||      |      }t        j
                  |t        j                  t        j                  t        j                  ||      |      |            }	y |      }
t        j                  t        j                  fv sJ 
t        j                  k(  r$t        j                  |t        j                        }t        j                  ||      }|S )Nr    r   r   r   r   r   mulsubr   r   )indexinputweight_compens_index_x_scale_x_zp_w_scale_weight_compotemp_biasr=   
bias_dtypebias_loaderinput_loaderw_scale_loaderweight_compens_loaderx_scale_loaderx_zp_loaders            rH   inner_fnz]register_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.epilogue_creator.<locals>.inner_fn  s)   $0$7E %(LL$FE49"I<0'5b'9H$/OE'56J'KH,ABV,WM $'77 #$)$,!" !)$D $'77 $ #$'GG(+,4,4)* ).%& %2	!"$D  $/(34H(I'1emmU^^5T'T T'T#-#?,/LL,NE'*wwtU';#'KrW   r(   r   r   rangesri   rg   c                 @     |       }t        j                  |      S rP   r   r   r   r   output_cast_loaderr   s     rH   inner_fn_cast_output_to_bf16zqregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.epilogue_creator.<locals>.inner_fn_cast_output_to_bf16      (:5(A'*||E<'H HrW   r   _create_constantsc                     |       } 	d|z  |t         j                        \  }}t        j                  ||z        |z   }
t         j                  k(  r 	ddt         j                        \  }}n 	ddt         j                        \  }}t        j
                  t        j                  ||      |      }t        j                  |
      S Ng      ?r   r      i   r   r   r   rounduint8minimummaximumr   r   scale
zero_pointr   	inv_scalevalqminqmaxclampedr   r   requant_input_loaders            rH   inner_fn_requantzeregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.epilogue_creator.<locals>.inner_fn_requant   s    (<U(C8I$'%K5==9" 5	: '*ii	0A&BZ&O#/5;;#>1B()3emm2&JD$ 2C(,c2&JD$ +.++ckk#t6Ld*S'*||G\'J JrW   r   r   r   r   r   r   int8make_loaderr   	Pointwiser5   r)   r
   get_device_or_errorloweringr   	functoolspartialr   r   )input_bufferr   
output_bufr   r  r   r   r   r   r  r   r   r   r   r:   r8   r=   r   o_scaler   r   r9   r   weight_compensr   r   s        @@@@@@@@@rH   r%   zKregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.epilogue_creator  s   +!MM!NN!KK!JJ	0      (4'?'?'A0>0J0J0L-)0)<)<)>)0)<)<)>&*&6&6&8&*+*.*:*:*<K3( 3(j &(\\#/#:#:#<"'--%-#/#8#8#:	&
  6>)B *D'Y*J
 (5>>91;1G1G1I.I *,'1'E'E'G&2)E'1':':'<	*JN  *)C *ekk5::-FFC3=3I3I3K0K" *,'1'E'E'G&2)2):):$4*/./2</@*"
 (2':':'<	*J  *)rW   )r   r   r   r         )   r   r   r   r   r  r  r#   r%   rk   )output_scaleoutput_zero_pointr   post_op_namepost_op_argspost_op_algorithmr=   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zCregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.<lambda>A  rt   rW   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zCregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.<lambda>B  rt   rW   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zCregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.<lambda>C  rt   rW   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zCregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.<lambda>D  rt   rW   )r   r  r  r  c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zCregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.<lambda>K      QWW->->qzz|-L rW   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zCregister_onednn_fusion_ops.<locals>.qlinear_unary.<locals>.<lambda>P  r  rW   qlinear_unaryrv   )*r   r   r  r)   r*   r   
isinstancer   r   r   r   r   ro   r   r   r   realizeallr   r   	get_numelInputsKernelunwrap_storage_for_inputConstantBufferrp   rq   tor   r   equal
zeros_liker   to_denser   r   r   r0   r   r-   ry   rz   r   )r   r   r   r   r   r   r=   r  r   r   r8   r9   r:   r"   r;   w_zp_tensorr>   r?   W_tensorweight_compens_tensorr%   rA   rw   rC   r   r  aten_mkldnn_qlinear_unarys    `` ` ```````           @@rH   r   z1register_onednn_fusion_ops.<locals>.qlinear_unary1  s   " !**,

: D: ZZ\F6{QR,-gr||4G}---''55LL>Y 6  !>7+;+;+=>> #7B/G7++-.&8T:TT8|
 ww22LL%++6V 3  dBLL1DzS(((ww22LLU[[9 3  >>#q(T*TT(
 | ww22LL%++6V 3  OOLLN~~5;;.:88>!!4
  gg//@CCEKKPww22LLEKK@t}} 3  "&4>>3CJ*,G!/6}V|0,FA} @@F)) (():):4==?)KL))$--/: ,FA}E ww001G1G1IJSSUH,1IIhkk%++6NTU,V)%&WW%@%@-*3358II &A &N
C* C*J ;;=U[[%**,EEEE#//< GT='4H$wdS!%T!1)9< '92 7|q $9$;!(&2!-!%!(&/ <%)F6N2-22< GT='4H$wdS	
 ! !))+qww/@/@@@@<<<<	M 88A!!
 $Ma 88>!! $Ma .< GT='4@$wdK+F 6{Qf&Ks&KV__5Fr5J&KLMrW   x2c                   	
  | j                         }j                         }t        |      t        |      k(  sJ t        |      dkD  r'|dk(  r"t        | d|d   g      } t        d|d   g      t        t        j
                        sYt              t        k(  sJ t        j                  j                  t        j                  t        j                        d      n^j                          t        d j                         D              rt        g       t        j                               dv sJ d	       Dt        j                  j                  t        j                  d
t        j                         d      |Dt        j                  j                  t        j                  d
t        j                         d      }t        t        j
                        sYt              t"        k(  sJ t        j                  j                  t        j                  t        j                         d      nj                          j                          |j                          |j%                         t        j                   k7  rt        t        j&                  j)                  |      t        j*                        rt        j                  j,                  |j/                            j1                  t        j                         }t        j                  j                  t        j                  |t        j                         |j/                               }|dk(  r
t        j                  t        j2                  fv rPj%                         t        j                  t        j2                  fv r j%                         
k7  r't5        
      nj%                         
k(  sJ d       j%                          j%                         nd g }t7               r&|dk(  r t9        | ||
      ^ }}} }t        t        j&                  j)                        t        j*                        rt        j;                         j<                        d
k(  rt        t        j&                  j)                  |      t        j*                        rst        j>                  t        j@                  t        j                  j,                  |j/                                  t        j                  j,                  |j/                                  rtC        || |      rt        j                  j,                  |j/                            }|jE                         }t        jF                  |j1                  t        j                        d
      }t        j                  j                  ||j/                         dz         	
 fd}tI        jJ                  ||	| ||gn	| ||gd u|g dng d       t        |      d
k(  s
tM               rRtO        	
||||
      }d |d<   |jQ                   !jR                  	| ||fn	| ||f|fi |       |j/                         t        j                  j,                  v sJ d d d d}d |d<   tU        d|	| ||gn	| ||g||      }t        |      dkD  r*|dk(  r%t        |g |d d |j                         d         }|S ) Nr   r   r    r   r   r   c              3   &   K   | ]	  }|d k(    ywr   r   r   s     rH   r   zEregister_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.<genexpr>  r   r   r   r   r   r   r   r   zCdtype of accum for qlinear post op sum should be the same as outputr   r   r   c                 f  	
 t         j                  t         j                  t         j                  t         j                  fv sJ | j                         j                         j                         j                         j                         
j                         d j                         
f
d}t        j                  | j                         t         j                  || j                               }dk7  rt        |      }t         j                  k(  rM|j                         fd}t        j                  |j                         ||j                               }|S t         j                  t         j                  fv rddlm |j                         		fd}t        j                  |j                         t         j                  t        j                  |t!              t#              	      |j                               }|S )
Nc                   
  |       } |       } d      } d      }t        j                  |t        j                        }| d   f} |      } |      }t        j                  t        j                  ||      |      }t        j
                  |t        j                  t        j                  t        j                  ||      |      |            }
y |      }	t        j                  t        j                  fv sJ t        j                  k(  r$t        j                  |	t        j                        }	t        j                  ||	      }t        j                  t        j                  fv sJ t        j                  k(  r$t        j                  |t        j                        }t        j                  ||      }|S )Nr   r    r   )r   r   _x2r   r   r   r   _weight_compensr   r   r=   r   r   r   r   r   x2_dtype	x2_loaderr   r   s             rH   r   z^register_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.epilogue_creator.<locals>.inner_fn  s   $0$7E"+E"2C'5b'9H$/OE %(LL$FE49"I<0'56J'KH.C 4/O $'77 #$)$,!" !)$D $'77 $ #$'GG(+,4,4)* ).%& %4	!"$D  $/(34H(I'1emmU^^5T'T T'T#-#?,/LL,NE'*wwtU'; $,u~~/N#NN#N'5>>9&)ll3&F#&774#5D#'KrW   r   ri   rg   c                 @     |       }t        j                  |      S rP   r   r   s     rH   r   zrregister_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.epilogue_creator.<locals>.inner_fn_cast_output_to_bf16=  r   rW   r   r   c                     |       } 	d|z  |t         j                        \  }}t        j                  ||z        |z   }
t         j                  k(  r 	ddt         j                        \  }}n 	ddt         j                        \  }}t        j
                  t        j                  ||      |      }t        j                  |t         j                        S r   r   r   s            rH   r  zfregister_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.epilogue_creator.<locals>.inner_fn_requantL  s    (<U(C8I$'%K5==9" 5	: '*ii	0A&BZ&O#/5;;#>1B()3emm2&JD$ 2C(,c2&JD$ +.++ckk#t6Ld*S'*||GU[['I IrW   r  r  )r  r   r  r   r  r   r   r   r   r  r   r   r8  r   r   r=   r   r  r   r   r   r^   r_   r   r  r0  r7  r   r   s        @@@@@@@@@@rH   r%   zLregister_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.epilogue_creator  s   +!MM!NN!KK!JJ	0      (4'?'?'A$&NN$4	0>0J0J0L-)0)<)<)>)0)<)<)>&*&6&6&8&*+*.*:*:*<K5( 5(n &(\\#/#:#:#<"'--%-#/#8#8#:	&
 &/)B * *(5*:	*J (5>>91;1G1G1I.I *,'1'E'E'G&2)E'1':':'<	*JN  *)C *ekk5::-FFC3=3I3I3K0J" *,'1'E'E'G&+kk)2):):$4*/./2</@*"
 (2':':'<	*J  *)rW   )r   r   r   r   r  r  r  )   r   r   r   r   r  r  r  r  )
r  r  r   other_scaleother_zpbinary_post_opr]   unary_post_opunary_post_op_argsunary_post_op_algorithmr=   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zDregister_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.<lambda>  rt   rW   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zDregister_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.<lambda>  rt   rW   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zDregister_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.<lambda>  rt   rW   )r   r  r  c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zDregister_onednn_fusion_ops.<locals>.qlinear_binary.<locals>.<lambda>  r  rW   r;  qlinear_binaryrv   )+r)   r*   r   r!  r   r   r   r   r   ro   r   r   r   r   r"  r#  r   r   r   r%  r&  r'  rp   rq   r(  r   r   r   r   
get_layoutsizer)  r*  r   r+  r   r   r0   r   r-   ry   rz   r   )"r   r   r   r   r   r   r0  r=   r  r   r   x2_scalex2_zpr\   r   r^   r_   r   r"   r;   x2_sizer,  r>   r?   r-  r.  r%   rA   rw   rC   r   r  r7  aten_mkldnn_qlinear_binarys"    `` ` `````    ```            @@@rH   rF  z2register_onednn_fusion_ops.<locals>.qlinear_binary_  sW   6 ZZ\FkkmGv;#g,...6{Q;%#7R,-"r72;/0gr||4G}---''55LL>Y 6  !>7+;+;+=>> #7B/G7++-.&8T:TT8|ww22LL%++6V 3  |ww22LL%++6V 3  dBLL1DzS(((ww22LLU[[9 3  
 OOLLN~~5;;.:88>!!4  gg//@CCEKKPww22LLEKK@t}} 3  e#MMNN$  lln(GG||~5
 &b,7<<>\9 ]9 ||~H-1-=)4J*,G "{e';3:}b<40FA}b @@F)) DOO-223q8"@@F)) (():):4==?)KL))$--/: .faG ww001G1G1IJH'002H,1IIhkk%++6NTU,V)%&WW%@%@-*3358II &A &N
J* J* J*X $//< GT='4L$wbRVW!%T!1)9  < '<5 7|q $9$;!(&2!- ("#.!&",'4,< <%)F6N3.33< GT='4L$wbRVW	
 ! !))+qww/@/@@@@<<<M
 #La . < GT='4D$wb$O+F 6{Q;%#7f&Ks&KV__5Fr5J&KLMrW   zmkl::_mkl_linearr!   packed_worig_wc                8   g }t               rMt        |ddg      }t        | ||      ^ }}} }t        || |      rt	        j
                  ||| ||gdddg       t        |      dk(  s
t               r'|j                  j                  | ||f|d |             |j                         t        j                  j                  v sJ |j                         t        j                  j                  v sJ d d	 d
}	t        d|| ||g||	      }
|t        |
|      }
|
S )Nr   r   r!   Tr   )r$   rk   )rl   
batch_sizec                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zGregister_onednn_fusion_ops.<locals>.mkl_packed_linear.<locals>.<lambda>      !2!21::<!@ rW   c                 X    t         j                  j                  | j                            S rP   rn   rr   s    rH   rs   zGregister_onednn_fusion_ops.<locals>.mkl_packed_linear.<locals>.<lambda>  rR  rW   )r   r   packed_linearrv   )r   r   r   r   r   r0   r*   r   ry   rz   rq   r   ro   rp   r   r   )r   rM  rN  r   rP  r"   r>   r{   r?   rw   rC   aten_mkl_linears              rH   mkl_packed_linearz5register_onednn_fusion_ops.<locals>.mkl_packed_linear  s>    /1#%#*6Aq6#:L29<3/Q< -VQE'33#"&1$(+,a& w<1$(=(?NN',,&16Tj -   ((*agg.?.????(AGG,=,==== A@! %>#&)"/% = ^FrW   rP   )%r   _C_has_mkldnn rK   r   r   mkldnn_linear_pointwiseLinearUnaryr7   binaryLinearBinaryonednnqlinear_pointwiseQLinearPointwisePT2EQLinearPointwiseBinaryPT2E_convolution_pointwise_convolution_pointwise_ _convolution_transpose_pointwiser   r   defaultqconv2d_pointwiser   r   boolr3   r   binary_tensorhas_mklmkl_mkl_linearMKLPackedLinearry   r   r   )cpu_needs_realized_inputsrV   ra   rd   ru   r   r   r   r   r   r   rF  rV  rU  r   r|   rL  r/  rK   s                @@@@@@rH   register_onednn_fusion_opsro  c   s   xx#5II..'!$0077	$
  %7II..55'!$1188	%
! %7II..'!$99@@	%
! &8II..55'!$??FF	&
" II33II44II==II..!!))II..%
! 
599++BB	C			 	 
D	6 
599++BBII	J			 	 		 
K	B 
599++CCJJ	K			 	 		 
L	B 
599++==	> ?	?	?	 ?	 
??	B 
599++==DD	EQU;	;	&;	+4;	9B;	 
F;	z 
599++LL	M			 	 
N	: 
40088	9&	&	&	 &	 	&	
 &	 &	 &	 &	 c&	 &	 &	 &	 &	  &	 &	  !&	 
:&	P 
599++==SW	X1	1	 %	1	
 1	 1	 1	 
Y1	f 
II..554

 
II..<<RV

F	F	 %	F	
 F	 F	 F	 F	



F	P 
599++==SW	X k	k	 %	k	
 k	 k	 k	 
Yk	Z	 
II..554

 
II..<<RV

, '~	~	 %	~	
 ~	 ~	 ~	 ~	



~	@
 880		))" %(88??	O &,,UYY]]-F-FGuyy}}889 00#0 "0 I&	0 :0d 	"";<rW   )NNNN))r
  typingr   r   torch.utils._pytreeutils_pytreer    torch._inductor.kernel.mm_commonr   rY  r   codegen.cpp_gemm_templater   !codegen.cpp_grouped_gemm_templater   codegen.cpp_utilsr
   r   r	  r   r   r   r   r   r   r   select_algorithmr   r   r   r   r   r   virtualizedr   r   r3   rI   _inductor_lowering_functionro  r   rW   rH   <module>r{     s       $ $ 4  6 E 8    
 R Q  
==I= I=@ 59  1@rW   