
    Vh                       U d dl mZ d dlZd dlZd dlmZ d dlmZmZ d dl	m
Z
 d dlmZ d dlZd dlZd dlmZ d dlmc mZ d dlmZ d dlmZmZmZmZmZmZmZmZ d d	lm Z m!Z! d d
l"m#Z#m$Z$ d dl%m&Z& d dl'm(Z( d dl)Zd dl*m+Z+m,Z,m-Z- d dl.m/Z/ ej`                  Z0dZ1d Z2d Z3 e4ddddd ddejj                         e4dddddd ddejj                  	       e4dd d!ejj                  "       e4dd#d$d%d&d!ejj                  '       e4d(d)d* ejj                  +       e4d,d-d.d/d!d!ejj                  0      gZ6d1 Z7d2 Z8d3 Z9d4 Z:d5 Z;d6 Z<d7 Z=d8 Z>d9 Z?d: Z@d; ZAd< ZBd= ZCd> ZDd? ZEd@ ZFdA ZGdB ZHdC ZIdD ZJdE ZKdF ZLdG ZMdH ZNdI ZOdJ ZPdK ZQdL ZRdM ZSdN ZTdO ZUdP ZVdQ ZWdR ZXdS ZYdT ZZdU Z[dV Z\dW Z]dX Z^dY Z_dZ Z`d[ Zad\ Zbd] Zcd^ Zdd_ Zed` Zfda Zgdb Zhdc Zidd Zjde Zkdf ZlddgZm	 	 ddhZn	 	 ddiZo	 	 ddjZp	 	 ddkZq	 	 ddlZrddmZsddnZtdo ZuddpZvddqZwddrZxds ZyddtZzdduZ{	 	 ddvZ|ddwZ}ddxZ~em eemdy      erenesetevewexeze{e|e}e~eqdzZe4d{e+f   ed|<   g Zd} Zg d~Zg dZ e
ee      D ]9  \  ZZ e4e de edddded!ejj                        Zej                  e       ; eD ]3  Z e4de d d d ed!ejj                        Zej                  e       5 dd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fg
Zddd!iiZe4ee4f   ed<   d!d!d!d!d!dZg dZ e
ee      D ]o  \  \  ZZZZ e4e de eddefdefdedej                  ed            Zej                  ei       Zej%                  e       ej                  e       q  G d de      Z G d d      Z G d de      Z G d d      Z G d dee      Z G d dee      ZddZddZd Zy)    )abstractmethodN)deepcopy)reducepartial)product)mul)
_reduction)TestCaseto_gpufreeze_rng_stateis_iterable	gradcheckgradgradcheckset_default_dtypeskipIfTorchDynamo)	TEST_CUDASM90OrLater)_get_numerical_jacobian_iter_tensors)Variable)_TensorOrTensors)CallableUnionAny)Sequenceh㈵>c                 r    t        | dd       }|#t        j                  t        | dd       dd      }|J |S )N	reductionsizeAverageTF)emit_warning)getattr
_Reductionlegacy_get_stringmresults     Q/home/dcms/DCMS/lib/python3.12/site-packages/torch/testing/_internal/common_nn.pyget_reductionr(   !   sD    QT*F~--ga.MtbghM    c                 >    t        | dd       }||S t        | dd       S )Nweightweights)r!   r$   s     r'   
get_weightr-   )   s*    Q$'F1i&&r)   Linear)
      ztorch::nn::LinearOptions(10, 8))   r/   c                     t        j                  | |d   j                               |d   j                  dd      j	                  dd      z   S )Nr      r1   r0   )torchmmtviewexpandip_s      r'   <lambda>r>   l   s>    UXXa1%:QqTYYq"=M=T=TUVXY=Z%Z r)   T{Gzt?)module_nameconstructor_argscpp_constructor_args
input_sizereference_fn	with_tf32tf32_precisiondefault_dtype)r/   r0   Fz+torch::nn::LinearOptions(10, 8).bias(false)no_biasc                 P    t        j                  | |d   j                               S Nr   )r5   r6   r7   r:   s      r'   r>   r>   w   s    UXXa1%: r)   )	r@   rA   rB   rC   descrD   rE   rF   rG   RReLU)r3      rM   F)r@   rC   	test_cudarG   皙?g?/torch::nn::RReLUOptions().lower(0.1).upper(0.9))r1   r1      with_up_downr@   rA   rB   rC   rK   rN   rG   FlattenrM      r1   rR   c                 .    t        j                  | d      S Nr3   )r5   flatten)r;   r=   s     r'   r>   r>      s    5==A#6 r)   )r@   rC   rD   rG   CrossMapLRN2d)rR   r?   MbP?rM   z>torch::nn::CrossMapLRN2dOptions(5).alpha(5e-3).beta(1e-3).k(2)rM   rW      r^   )r@   rA   rB   rC   check_gradgradcheck_batched_gradrG   c                      t        t        | d      } t        j                  |      j                  |  j                         S rY   )r   r   r5   randpermr8   double)sizetotals     r'   _rand_tensor_non_equalrf      s5    3a E%5>>% %%t,3355r)   c                 B      G  fddt         j                        }|S )Nc                       e Zd Z fdZy))wrap_functional.<locals>.FunctionalModulec                      |i S N )selfargsfnkwargss     r'   forwardz1wrap_functional.<locals>.FunctionalModule.forward   s    t&v&&r)   N)__name__
__module____qualname__rq   )ro   rp   s   r'   FunctionalModuleri      s    	'r)   ru   )nnModule)ro   rp   ru   s   `` r'   wrap_functionalrx      s    '299 ' r)   c            
           t        j                  dd       t        dt         fd      dd d d fdd	t         j                  
      S )Nr/   PoissonNLLLoss_no_reducec                 R    t        j                  | j                  |       d      S Nnoner   )Fpoisson_nll_losstype_asr;   r7   s    r'   r>   z/poissonnllloss_no_reduce_test.<locals>.<lambda>       a((AIIaLFK r)   zaF::poisson_nll_loss(i, t.to(i.options()), F::PoissonNLLLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S Nr/   r5   randrl   r)   r'   r>   z/poissonnllloss_no_reduce_test.<locals>.<lambda>   s    B+ r)   _get_input()r   c                 H    | j                         j                  |       z
  S rk   )expr   r;   r=   r7   s     r'   r>   z/poissonnllloss_no_reduce_test.<locals>.<lambda>   s    1557QUU1X#5 r)   Ffullnameconstructorcpp_function_callinput_fncpp_var_maprD   picklerG   r5   randndictrx   rc   r7   s   @r'   poissonnllloss_no_reduce_testr      sM    BA+#KMj+(q15ll
$ 
$r)   c                      t        t        j                  dd      j                  d      j	                  t        j
                               t        dt         fd      dd d d	 fd
ddt        j
                  	      S )N   r/   r   BCELoss_no_reducec                 R    t        j                  | j                  |       d      S r|   r   binary_cross_entropyr   r   s    r'   r>   z(bceloss_no_reduce_test.<locals>.<lambda>       a,,Q		!O r)   iF::binary_cross_entropy(i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S Nr   r/   y&1?v?r5   r   clamp_rl   r)   r'   r>   z(bceloss_no_reduce_test.<locals>.<lambda>       B+226:F r)   r   r   c                 `    | j                         z  dz
  d| z
  j                         z  z    S rY   logr   s     r'   r>   z(bceloss_no_reduce_test.<locals>.<lambda>   ,    Q[AEa!e[[]3J%J#K r)   FgǺF?	r   r   r   r   r   rD   r   	precisionrG   )r   r5   r   gttorc   r   rx   r   s   @r'   bceloss_no_reduce_testr      so    R$''*--ell;<A$#OQnF(q1Kll$ $r)   c            
           t        j                  d      j                  d      j                  t         j                         t        dt         fd      dd d d fd	d
t         j                        S )Nrl   r   BCELoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r|   r   r   s    r'   r>   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>   r   r)   r   c                  L    t        j                  d      j                  dd      S Nrl   r   r   r   rl   r)   r'   r>   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>       B..vzB r)   r   r   c                 `    | j                         z  dz
  d| z
  j                         z  z    S rY   r   r   s     r'   r>   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>   r   r)   Fr   )r5   r   r   r   rc   r   rx   r   s   @r'   bceloss_no_reduce_scalar_testr      sd    B1  .A+#OQnB(q1Kll
$ 
$r)   c                  v    t        t        j                  ddt        j                        j	                  d      j                  t        j                               t        j                  dt        j                        t        dt         fd      dd d	 d
 fdddt        j                  	      S )Nr   r/   dtyper   BCELoss_weights_no_reducec                 r    t        j                  | j                  |       j                  |       d      S Nr}   r+   r   r   r;   r7   r,   s    r'   r>   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   .    a,,Q		!4;OOA4FRXZ r)   zF::binary_cross_entropy(i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S r   r   rl   r)   r'   r>   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   r   r)   r   r   c                 f    | j                         z  dz
  d| z
  j                         z  z    z  S rY   r   )r;   r<   r%   r7   r,   s      r'   r>   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   s2    q1557{a!eA{{}5L'L%MPW%W r)   Fa2U0*3?r   )	r   r5   r   rc   r   r   r   r   rx   r7   r,   s   @@r'   bceloss_weights_no_reduce_testr      s    R5<<8;;A>AA%,,OPAjj5<<0G,#Z[x G(qWEWll r)   c            
      @    t        j                  d      j                  d      j                  t         j                         t        j
                  dt         j                        t        dt         fd      dd dd	  fd
dt         j                        S )Nrl   r   r    BCELoss_weights_no_reduce_scalarc                 r    t        j                  | j                  |       j                  |       d      S r   r   r   s    r'   r>   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>   r   r)   zF::binary_cross_entropy(
            i, t.to(i.options()),
            F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))r   r   c                  L    t        j                  d      j                  dd      S r   r   rl   r)   r'   r>   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>   r   r)   c                 f    | j                         z  dz
  d| z
  j                         z  z    z  S rY   r   r;   r=   r7   r,   s     r'   r>   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>   s1    Q[AEa!e[[]3J%J#Kg#U r)   F)r   r   r   r   r   rD   r   rG   )r5   r   r   r   rc   r   r   rx   r   s   @@r'   %bceloss_weights_no_reduce_scalar_testr      s    B1  .Ajj5<<0G3#Z[k )qWEBUll r)   c                  0    t        t        j                  dd      j                  d      j	                  t        j
                              t        j                          t        dt        fd      dd dd	 fd
ddt        j
                  	      S )Nr   r/   r   BCEWithLogitsLoss_legacy_enumc                 R    t        j                  | j                  |       d      S )NF)r   r    binary_cross_entropy_with_logitsr   r   s    r'   r>   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>	  s    a88AIIaLQVW r)   F::binary_cross_entropy_with_logits(
            i, t.to(i.options()), F::BinaryCrossEntropyWithLogitsFuncOptions().reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S r   r   rl   r)   r'   r>   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>  r   r)   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S rY   r   r;   r=   sigmoidr7   s     r'   r>   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>  :    Q)9%9QUq7ST:~FZFZF\<\%\#] r)   F	r   r   r   r   r   rD   r_   r   rG   
r   r5   r   r   r   rc   rv   Sigmoidr   rx   r   r7   s   @@r'   "bce_with_logistic_legacy_enum_testr     sz    R$''*--ell;<AjjlG0#WYkF(q1]ll r)   c                  0    t        t        j                  dd      j                  d      j	                  t        j
                              t        j                          t        dt        fd      dd dd	 fd
ddt        j
                  	      S )Nr   r/   r   BCEWithLogitsLoss_no_reducec                 R    t        j                  | j                  |       d      S r|   r   r   s    r'   r>   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>      a88AIIaLTZ[ r)   r   c                  N    t        j                  dd      j                  dd      S r   r   rl   r)   r'   r>   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>  r   r)   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S rY   r   r   s     r'   r>   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>   r   r)   Fr   r   r   s   @@r'    bce_with_logistic_no_reduce_testr     sz    R$''*--ell;<AjjlG.#[]kF(q1]ll r)   c                      t        j                  d      j                  d      j                  t         j                        t        j                          t        dt        fd      dd dd fd	d
d
t         j                  	      S )Nrl   r   "BCEWithLogitsLoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r|   r   r   s    r'   r>   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>-  r   r)   r   c                  L    t        j                  d      j                  dd      S r   r   rl   r)   r'   r>   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>0  r   r)   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S rY   r   r   s     r'   r>   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>2  r   r)   Fr   )	r5   r   r   r   rc   rv   r   r   rx   r   s   @@r'   'bce_with_logistic_no_reduce_scalar_testr   '  sr    B1  .AjjlG5#[]kB(q1]ll r)   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nr/   r   KLDivLoss_with_target_no_reducec                 R    t        j                  | j                  |       d      S r|   r   kl_divr   r   s    r'   r>   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>>      ahhq!))A,&A r)   NF::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(torch::kNone))c                  J    t        j                  dd      j                         S r   r5   r   r   rl   r)   r'   r>   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>@      B+//1 r)   r   r   c                 D    t        d   | j                  |       d      S N	KLDivLossr}   r~   loss_reference_fnsr   r   s     r'   r>   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>B  !    {+Aqyy|vN r)   TF	r   r   r   r   r   rD   supports_forward_adr   rG   r5   r   rc   r   rx   r   s   @r'   $kldivloss_with_target_no_reduce_testr   9  sV    

2r.A2#ACj1(q1O ll$ $r)   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nr/   r   KLDivLoss_no_reducec                 R    t        j                  | j                  |       d      S r|   r   r   s    r'   r>   z*kldivloss_no_reduce_test.<locals>.<lambda>N  r   r)   r   c                  J    t        j                  dd      j                         S r   r   rl   r)   r'   r>   z*kldivloss_no_reduce_test.<locals>.<lambda>P  r   r)   r   r   c                 D    t        d   | j                  |       d      S r   r   r   s     r'   r>   z*kldivloss_no_reduce_test.<locals>.<lambda>R  r   r)   TFr   r   r   s   @r'   kldivloss_no_reduce_testr  I  sV    

2r.A&#ACj1(q1O ll r)   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nrl   r   KLDivLoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r|   r   r   s    r'   r>   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>_  r   r)   r   c                  H    t        j                  d      j                         S Nrl   r   rl   r)   r'   r>   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>a      B++- r)   r   r   c                 D    t        d   | j                  |       d      S r   r   r   s     r'   r>   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>c  r   r)   TFr   r   r   s   @r'   kldivloss_no_reduce_scalar_testr  Z  sT    

2U\\*A-#ACj-(q1O ll$ $r)   c                       t        j                  ddt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nr/   r   #KLDivLoss_with_log_target_no_reducec                 T    t        j                  | j                  |       dd      S Nr}   T)r   
log_targetr   r   s    r'   r>   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>o      ahhq!))A,&TR r)   _F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(torch::kNone).log_target(true))c                  J    t        j                  dd      j                         S r   r   rl   r)   r'   r>   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>q  r   r)   r   r   c                 D    t        d   | j                  |       d      S NKLDivLoss_log_targetr}   r~   r   r   s     r'   r>   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>s  #    56q!))A,RXY r)   TFr   r5   r   rc   r   r   rx   r   s   @r'   (kldivloss_with_log_target_no_reduce_testr  j  s_    

2r.224A6#RT{1(q1Z ll$ $r)   c                       t        j                  ddt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nr/   r   KLDivLoss_no_reduce_log_targetc                 T    t        j                  | j                  |       dd      S r  r   r   s    r'   r>   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>  r  r)   r  c                  J    t        j                  dd      j                         S r   r   rl   r)   r'   r>   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>  r   r)   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r'   r>   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>  r  r)   TFr   r  r   s   @r'   #kldivloss_no_reduce_log_target_testr  z  s_    

2r.224A1#RT{1(q1Z ll r)   c                       t        j                  dt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nrl   r   %KLDivLoss_no_reduce_scalar_log_targetc                 T    t        j                  | j                  |       dd      S r  r   r   s    r'   r>   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r  r)   r  c                  H    t        j                  d      j                         S r  r   rl   r)   r'   r>   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r	  r)   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r'   r>   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r  r)   TFr   r  r   s   @r'   *kldivloss_no_reduce_scalar_log_target_testr%    s]    

2U\\*..0A8#RT{-(q1Z ll$ $r)   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrM   rW   r1   r   L1Loss_no_reducec                 R    t        j                  | j                  |       d      S r|   r   l1_lossr   r   s    r'   r>   z'l1loss_no_reduce_test.<locals>.<lambda>      aii199Q<6B r)   PF::l1_loss(i, t.to(i.options()), F::L1LossFuncOptions().reduction(torch::kNone))c                  0    t        j                  ddd      S NrM   rW   r1   r5   r   rl   r)   r'   r>   z'l1loss_no_reduce_test.<locals>.<lambda>      Q1- r)   r   r   c                 H    | j                  |       z
  j                         S rk   r   absr   s     r'   r>   z'l1loss_no_reduce_test.<locals>.<lambda>      A		!$4#9#9#; r)   TFr   r5   r   rc   r   rx   r   s   @r'   l1loss_no_reduce_testr6    sU    Aq!5<<0A##BDl-(q1; ll
$ 
$r)   c            
           t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fddd      S )NrM   rW   r1   r   L1Loss_no_reduce_complexc                 R    t        j                  | j                  |       d      S r|   r)  r   s    r'   r>   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  r+  r)   r,  c                  P    t        j                  dddt         j                        S )NrM   rW   r1   r   )r5   r   cdoublerl   r)   r'   r>   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  s    Q1EMMB r)   r   r   c                 H    | j                  |       z
  j                         S rk   r2  r   s     r'   r>   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  r4  r)   TF)r   r   r   r   r   rD   r   r   )r5   r   r;  r   rx   r   s   @r'   l1loss_no_reduce_complex_testr=    sN    Aq!5==1A+#BDlB(q1; 	 	r)   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nrl   r   L1Loss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r|   r)  r   s    r'   r>   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>  r+  r)   r,  c                  ,    t        j                  d      S r  r/  rl   r)   r'   r>   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>      R r)   r   r   c                 H    | j                  |       z
  j                         S rk   r2  r   s     r'   r>   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>  r4  r)   TFr   r5  r   s   @r'   l1loss_no_reduce_scalar_testrD    sQ    Bell+A*#BDl((q1; ll
$ 
$r)   c                      d} t        j                  | dt         j                  it        dt	        fd      d| ddfdd	d
t         j                  	      S )NrV   r   MSELoss_no_reducec                 R    t        j                  | j                  |       d      S r|   r   mse_lossr   r;   targets    r'   r>   z(mseloss_no_reduce_test.<locals>.<lambda>      ajjFNN1$5H r)   WF::mse_loss(i, target.to(i.options()), F::MSELossFuncOptions().reduction(torch::kNone))r   rJ  c                 ,    | z
  j                  d      S NrM   powr;   r=   rK  s     r'   r>   z(mseloss_no_reduce_test.<locals>.<lambda>      AJ#3#3A#6 r)   TF	r   r   r   rC   r   rD   r   r   rG   r5  rC   rK  s    @r'   mseloss_no_reduce_testrV    sY    J[[*9ELL9F$#HJs(F;6 ll
$ 
$r)   c                      d} t        j                  | t         j                        t        dt	        fd      d| ddfdd	d
t         j                  	      S )Nrl   r   MSELoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r|   rH  rJ  s    r'   r>   z/mseloss_no_reduce_scalar_test.<locals>.<lambda>  rL  r)   rM  r   rJ  c                 ,    | z
  j                  d      S rO  rP  rR  s     r'   r>   z/mseloss_no_reduce_scalar_test.<locals>.<lambda>  rS  r)   TFrT  r5  rU  s    @r'   mseloss_no_reduce_scalar_testr[    sV    J[[5<<8F+#HJs(F;6 ll
$ 
$r)   c            
      (    t        t        j                  d      j                         j	                  d      j                         j                               ddi t        dt         fd      dd d	d
 fddt        j                        S )Nr   r/   r   r}   NLLLoss_no_reducec                 t    t        j                  | j                  |       j                         d         S Nr   r~   r   nll_lossr   longr;   rp   r7   s    r'   r>   z(nllloss_no_reduce_test.<locals>.<lambda>  *    ajjAIIaL$5$5$76+CVW r)   pF::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().reduction(torch::kNone))c                  J    t        j                  dd      j                         S Nr   r/   r   rl   r)   r'   r>   z(nllloss_no_reduce_test.<locals>.<lambda>  r   r)   r   r   c                 ^    t        d   | j                  |       j                         fi S NNLLLossr   r   rb  r;   r=   rp   r7   s     r'   r>   z(nllloss_no_reduce_test.<locals>.<lambda>  -    y)!QYYq\->->-@KFK r)   Fr   
r   r5   emptyuniform_r   floorrb  r   rx   rc   rp   r7   s   @@r'   nllloss_no_reduce_testrs    s    R))+//399;@@BCA6"F$#WYg1(q1Lll$ $r)   c            
      *    t        t        j                  d      j                         j	                  d      j                         j                               ddd t        dt         fd      dd	 d
d fddt        j                        S )Nr   r/   rM   r}   ignore_indexr   NLLLoss_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S Nrv  r   ru  r   ra  r   rb  intstrrc  s    r'   r>   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  @    ajjAIIaL$5$5$7c&Q_J`Fa+.vk/B+CE r)   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().ignore_index(2).reduction(torch::kNone))c                  J    t        j                  dd      j                         S rg  r   rl   r)   r'   r>   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  r   r)   r   r   c                 ^    t        d   | j                  |       j                         fi S ri  rk  rl  s     r'   r>   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  rm  r)   Fr   rn  rr  s   @@r'   #nllloss_no_reduce_ignore_index_testr    s    R))+//399;@@BCA:;&)QF1#EFw1(q1Lll$ $r)   c            
      X    t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dd dd	 fd
dt        j                        S )Nr   r/   c                 ,    j                  |       ddS r   r   r;   r+   s    r'   rp   z.nllloss_no_reduce_weights_test.<locals>.kwargs       ..+&AAr)   NLLLoss_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rk   r`  rc  s    r'   r>   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  ,    ajjAIIaL$5$5$7E6!9E r)   F::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone))c                  h    t        j                  dd      j                  d      j                         S Nr   r/   {Gz?r5   r   addr   rl   r)   r'   r>   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  %    B+//599; r)   r   r;   r7   r+   c                 j    t        d   | j                  |       j                         fi  |       S ri  rk  rl  s     r'   r>   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  1    y)!QYYq\->->-@NF1IN r)   Fr   r   r5   ro  rp  r   rq  rb  r   r   rx   rc   rp   r7   r+   s   @@@r'   nllloss_no_reduce_weights_testr  
  s    R))+//399;@@BCAZZ^FB ,#EG_ <(qFCOll$ $r)   c            
      X    t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dd dd	 fd
dt        j                        S )Nr   r/   c                 .    j                  |       dddS )Nr}   rM   r+   r   rv  r  r  s    r'   rp   z;nllloss_no_reduce_weights_ignore_index_test.<locals>.kwargs$  s     ..+& !# 	#r)   &NLLLoss_no_reduce_weights_ignore_indexc                     t        j                  | j                  |       j                         fi  | j                        S rk   )r   ra  r   rb  datarc  s    r'   r>   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>+  s0    ajjAIIaL$5$5$7J6!&&>J r)   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone).ignore_index(2))c                  h    t        j                  dd      j                  d      j                         S r  r  rl   r)   r'   r>   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>/  r  r)   r   r  c                 j    t        d   | j                  |       j                         fi  |       S ri  rk  rl  s     r'   r>   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>1  r  r)   Fr   r  r  s   @@@r'   +nllloss_no_reduce_weights_ignore_index_testr     s    R))+//399;@@BCAZZ^F# 9#JLo <(qFCOll$ $r)   c            
          t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dt        j                  ddt        j                        j                  d      j                         d	d
 fddt        j                        S )Nr   r/   c                 .    j                  |       dddS )Nr}   r4   r  r  r  s    r'   rp   z?nllloss_no_reduce_weights_ignore_index_neg_test.<locals>.kwargs;  s     ..+& "$ 	$r)   *NLLLoss_no_reduce_weights_ignore_index_negc                 x    t        j                  | j                  |       j                         fi  |       S rk   r`  rc  s    r'   r>   zAnllloss_no_reduce_weights_ignore_index_neg_test.<locals>.<lambda>B  r  r)   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone).ignore_index(-1))r   r  r   r  c                 j    t        d   | j                  |       j                         fi  |       S ri  rk  rl  s     r'   r>   zAnllloss_no_reduce_weights_ignore_index_neg_test.<locals>.<lambda>H  r  r)   F)r   r   r   inputr   rD   r   rG   )r   r5   ro  rp  r   rq  rb  r   r   rx   rc   r  r   r  s   @@@r'   /nllloss_no_reduce_weights_ignore_index_neg_testr  7  s    R))+//399;@@BCAZZ^F$ =#EGp jjRu||488>BBD(qFCOll$ $r)   c            
          t        t        j                  ddd      j                  d      j	                         j                               ddi t        dt         fd      dd	 d
d fddt        j                        S )NrM   rR   rW   r   r}   NLLLoss2d_no_reducec                 t    t        j                  | j                  |       j                         d         S r_  r`  rc  s    r'   r>   z*nllloss2d_no_reduce_test.<locals>.<lambda>T  rd  r)   re  c                  N    t        j                  dddd      j                         S NrM   rW   rR   r   rl   r)   r'   r>   z*nllloss2d_no_reduce_test.<locals>.<lambda>W      Aq!Q/335 r)   r   r   c                 ^    t        d   | j                  |       j                         fi S N	NLLLossNdrk  rl  s     r'   r>   z*nllloss2d_no_reduce_test.<locals>.<lambda>Y  -    {+Aqyy|/@/@/BMfM r)   Fr   	r   r5   r   r   rq  rb  r   rx   rc   rr  s   @@r'   nllloss2d_no_reduce_testr  N  s~    Aq!$((+11388:;A6"F&#WYg5(q1Nll$ $r)   c            
          t        t        j                  ddd      j                  d      j	                         j                               ddd t        dt         fd      d	d
 dd fddt        j                        S )NrM   rR   rW   r3   r}   ru   NLLLoss2d_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S ry  rz  rc  s    r'   r>   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>e  r}  r)   F::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().ignore_index(1).reduction(torch::kNone))c                  N    t        j                  dddd      j                         S r  r   rl   r)   r'   r>   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>i  r  r)   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rk  rl  s     r'   r>   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>k  r  r)   Fr   r  rr  s   @@r'   %nllloss2d_no_reduce_ignore_index_testr  _  s    Aq!$((+11388:;A:;&)QF3#EFw5(q1Nll$ $r)   c            
      @    t        t        j                  ddd      j                  d      j	                         j                               t        j                  d      fd t        dt         fd      dd d	d
 fddt        j                        S )NrM   rR   rW   c                 ,    j                  |       ddS r   r  r  s    r'   rp   z0nllloss2d_no_reduce_weights_test.<locals>.kwargsu  r  r)   NLLLoss2d_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rk   r`  rc  s    r'   r>   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>{  r  r)   r  c                  N    t        j                  dddd      j                         S r  r   rl   r)   r'   r>   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>  r  r)   r   r  c                 j    t        d   | j                  |       j                         fi  |       S r  rk  rl  s     r'   r>   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>  1    {+Aqyy|/@/@/BPfQiP r)   Fr   r  r  s   @@@r'    nllloss2d_no_reduce_weights_testr  q  s    Aq!$((+11388:;AZZ]FB .#EG_ 6(qFCQll$ $r)   c            
          t        t        j                  ddddd      j                  d      j	                         j                               ddi t        dt         fd      dd	 d
d fddt        j                        S )NrM   rR   rW   r   r}   NLLLossNd_no_reducec                 t    t        j                  | j                  |       j                         d         S r_  r`  rc  s    r'   r>   z*nlllossNd_no_reduce_test.<locals>.<lambda>  rd  r)   re  c                  R    t        j                  dddddd      j                         S r  r   rl   r)   r'   r>   z*nlllossNd_no_reduce_test.<locals>.<lambda>  "    Aq!Q1599; r)   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rk  rl  s     r'   r>   z*nlllossNd_no_reduce_test.<locals>.<lambda>  r  r)   Fr   r  rr  s   @@r'   nlllossNd_no_reduce_testr    s    Aq!Q*..q1779>>@AA6"F&#WYg;(q1Nll$ $r)   c            
          t        t        j                  ddddd      j                  d      j	                         j                               ddd t        dt         fd      d	d
 dd fddt        j                        S )NrM   rR   rW   r3   r}   ru   NLLLossNd_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S ry  rz  rc  s    r'   r>   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r}  r)   r  c                  R    t        j                  dddddd      j                         S r  r   rl   r)   r'   r>   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r  r)   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rk  rl  s     r'   r>   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r  r)   Fr   r  rr  s   @@r'   %nlllossNd_no_reduce_ignore_index_testr    s    Aq!Q*..q1779>>@AA:;&)QF3#EFw;(q1Nll$ $r)   c            
      D    t        t        j                  ddddd      j                  d      j	                         j                               t        j                  d      fd t        dt         fd      dd d	d
 fddt        j                        S )NrM   rR   rW   c                 ,    j                  |       ddS r   r  r  s    r'   rp   z0nlllossNd_no_reduce_weights_test.<locals>.kwargs  r  r)   NLLLossNd_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rk   r`  rc  s    r'   r>   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r)   r  c                  R    t        j                  dddddd      j                         S r  r   rl   r)   r'   r>   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r)   r   r  c                 j    t        d   | j                  |       j                         fi  |       S r  rk  rl  s     r'   r>   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r)   Fr   r  r  s   @@@r'    nlllossNd_no_reduce_weights_testr    s    Aq!Q*..q1779>>@AAZZ]FB .#EG_ <(qFCQll$ $r)   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrM   rW   r1   r   SmoothL1Loss_no_reducec                 R    t        j                  | j                  |       d      S r|   r   smooth_l1_lossr   r   s    r'   r>   z-smoothl1loss_no_reduce_test.<locals>.<lambda>      a&&q!))A,&I r)   jF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone))c                  0    t        j                  ddd      S r.  r/  rl   r)   r'   r>   z-smoothl1loss_no_reduce_test.<locals>.<lambda>  r0  r)   r   r   c                 D    t        d   | j                  |       d      S NSmoothL1Lossr}   r~   r   r   s     r'   r>   z-smoothl1loss_no_reduce_test.<locals>.<lambda>  !    ~.q!))A,&Q r)   TFr   r5  r   s   @r'   smoothl1loss_no_reduce_testr    s[    Aq!5<<0A)#IK[-(q1R ll$ $r)   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nrl   r   SmoothL1Loss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r|   r  r   s    r'   r>   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  r  r)   r  c                  ,    t        j                  d      S r  r/  rl   r)   r'   r>   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  rB  r)   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r'   r>   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  r  r)   TFr   r5  r   s   @r'   "smoothl1loss_no_reduce_scalar_testr    sW    Bell+A0#IK[((q1R ll$ $r)   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrM   rW   r1   r   SmoothL1Loss_betac                 T    t        j                  | j                  |       dd      S )Nr}         ?r   betar  r   s    r'   r>   z(smoothl1loss_beta_test.<locals>.<lambda>  s     a&&q!))A,&sS r)   zoF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone), 0.5)c                  0    t        j                  ddd      S r.  r/  rl   r)   r'   r>   z(smoothl1loss_beta_test.<locals>.<lambda>  r0  r)   r   r   c                 F    t        d   | j                  |       dd      S )Nr  r}   r  r  r   r   s     r'   r>   z(smoothl1loss_beta_test.<locals>.<lambda>  s$    ~.q!))A,&WZ[ r)   TFr   r5  r   s   @r'   smoothl1loss_beta_testr    s[    Aq!5<<0A$#SU`-(q1\ ll$ $r)   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrM   rW   r1   r   SmoothL1Loss_zero_betac                 T    t        j                  | j                  |       dd      S )Nr}   r   r  r  r   s    r'   r>   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  s     a&&q!))A,&qQ r)   zmF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone), 0)c                  0    t        j                  ddd      S r.  r/  rl   r)   r'   r>   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  r0  r)   r   r   c                 F    t        d   | j                  |       dd      S )Nr  r}   r   r  r   r   s     r'   r>   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  s$    ~.q!))A,&WXY r)   TFr   r5  r   s   @r'   smoothl1loss_zero_beta_testr    s[    Aq!5<<0A)#QS^-(q1Z ll$ $r)   c                       t        j                  ddd       t        dt         fd      dd d d	 fd
ddt         j                  	      S )NrM   rW   r1   HuberLoss_deltac                 T    t        j                  | j                  |       dd      S )Nr}   r  r   delta)r   
huber_lossr   r   s    r'   r>   z&huberloss_delta_test.<locals>.<lambda>	  s    all1aiilfCP r)   znF::huber_loss(
            i, t.to(i.options()), F::HuberLossFuncOptions().reduction(torch::kNone).delta(0.5))c                  0    t        j                  ddd      S r.  r/  rl   r)   r'   r>   z&huberloss_delta_test.<locals>.<lambda>  r0  r)   r   r   c                 F    t        d   | j                  |       dd      S )N	HuberLossr}   r  r  r   r   s     r'   r>   z&huberloss_delta_test.<locals>.<lambda>  s$    {+Aqyy|vUXY r)   TFr   r   r   s   @r'   huberloss_delta_testr     sU    Aq!A"#PRc-(q1Z ll$ $r)   c                       t        j                  d      j                          t        dt	         fd      dd d d fdd	d
d
	      S )Nrl   !MultiLabelMarginLoss_0d_no_reducec                 n    t        j                  | j                  |       j                         d      S r|   r   multilabel_margin_lossr   rb  r   s    r'   r>   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>  (    a..q!))A,2C2C2EQWX r)   F::multilabel_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultilabelMarginLossFuncOptions().reduction(torch::kNone))c                  ,    t        j                  d      S r  r/  rl   r)   r'   r>   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>  rB  r)   r   r   c                 t    t        d   | j                  j                  |       j                         d      S NMultiLabelMarginLossr}   r~   r   r  r   rb  r   s     r'   r>   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>  1    56q!&&..:K:P:P:R^de r)   TF)	r   r   r   r   r   rD   check_sum_reductionr_   r   )r5   zerosrb  r   rx   r   s   @r'   &multilabelmarginloss_0d_no_reduce_testr    sU    BA4#XZt((q1f  r)   c                      t        t        j                  d      j                  d      j	                         j                                t        dt         fd      dd d d fdd	d
d
t        j                  
      S )Nr/   !MultiLabelMarginLoss_1d_no_reducec                 n    t        j                  | j                  |       j                         d      S r|   r  r   s    r'   r>   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>+  r  r)   r  c                  ,    t        j                  d      S r   r/  rl   r)   r'   r>   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>.  rB  r)   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r
  r  r   s     r'   r>   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>0  r  r)   TF
r   r   r   r   r   rD   r  r_   r   rG   r  r   s   @r'   &multilabelmarginloss_1d_no_reduce_testr  &  su    B##B'--/4467A4#XZt((q1f ll$ $r)   c                  L    t        t        j                  t        j                  dd      j	                  d      j                  d      j                         j                         d             t        dt         fd      d	d
 d d fddddt        j                  
      S )NrR   r/   g         r4   minMultiLabelMarginLoss_index_negc                 n    t        j                  | j                  |       j                         d      S r|   r  r   s    r'   r>   z5multilabelmarginloss_index_neg_test.<locals>.<lambda>=  r  r)   r  c                  .    t        j                  dd      S NrR   r/   r/  rl   r)   r'   r>   z5multilabelmarginloss_index_neg_test.<locals>.<lambda>@      Q+ r)   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r
  r  r   s     r'   r>   z5multilabelmarginloss_index_neg_test.<locals>.<lambda>B  r  r)   TFr  )r   r5   clampr   r  r   rq  rb  r   rx   rc   r   s   @r'   #multilabelmarginloss_index_neg_testr#  8  s    UZZ2.2237;;B?EEGLLNTVWXA1#XZt+(q1f ll$ $r)   c                      t        t        j                  dd      j                  d      j	                         j                                t        dt         fd      dd d d fd	d
ddt        j                  
      S )NrR   r/   MultiLabelMarginLoss_no_reducec                 n    t        j                  | j                  |       j                         d      S r|   r  r   s    r'   r>   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>O  r  r)   r  c                  .    t        j                  dd      S r  r/  rl   r)   r'   r>   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>R  r   r)   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r
  r  r   s     r'   r>   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>T  r  r)   TFr  r  r   s   @r'   #multilabelmarginloss_no_reduce_testr)  J  sx    Ar"&&r*002779:A1#XZt+(q1f ll$ $r)   c                  >    t        t        j                  d      j                  d      j	                  t        j
                        j                  d      j                  d             t        dt         fd      dd d	 d
 fdddt        j
                  	      S )Nr/   r   rM   r3   HingeEmbeddingLoss_no_reducec                 R    t        j                  | j                  |       d      S r|   r   hinge_embedding_lossr   r   s    r'   r>   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>a  r   r)   zvF::hinge_embedding_loss(
            i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().reduction(torch::kNone))c                  ,    t        j                  d      S r   r/  rl   r)   r'   r>   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>d  rB  r)   r   r   c                 D    t        d   | j                  |       d      S )NHingeEmbeddingLossr}   r~   r   r   s     r'   r>   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>f  s#    34Q		!PVW r)   TF	r   r   r   r   r   rD   r  r   rG   
r   r5   r   r   r   rc   mul_subr   rx   r   s   @r'   !hingeembeddingloss_no_reduce_testr6  \  s    R##A&))%,,7<<Q?CCAFGA/#OQa((q1X ll$ $r)   c                  >    t        t        j                  d      j                  d      j	                  t        j
                        j                  d      j                  d             t        dt         fd      dd d	 d
 fdddt        j
                  	      S )Nr/   r   rM   r3   #HingeEmbeddingLoss_margin_no_reducec                 T    t        j                  | j                  |       dd      S Nr  r}   marginr   r-  r   s    r'   r>   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>r  s!    a,,Q		!STZ[ r)   zF::hinge_embedding_loss(
            i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().margin(0.5).reduction(torch::kNone))c                  ,    t        j                  d      S r   r/  rl   r)   r'   r>   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>u  rB  r)   r   r   c                 F    t        d   | j                  |       dd      S )Nr1  r  r}   r;  r   r   s     r'   r>   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>w  s%    34Q		!S\bc r)   TFr2  r3  r   s   @r'   (hingeembeddingloss_margin_no_reduce_testr?  m  s    R##A&))%,,7<<Q?CCAFGA6#[]m((q1d ll$ $r)   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )NrR   r   SoftMarginLoss_no_reducec                 R    t        j                  | j                  |       d      S r|   )r   soft_margin_lossr   r   s    r'   r>   z/softmarginloss_no_reduce_test.<locals>.<lambda>  r   r)   znF::soft_margin_loss(
            i, t.to(i.options()), F::SoftMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S )NrR   r/  rl   r)   r'   r>   z/softmarginloss_no_reduce_test.<locals>.<lambda>  s    Q* r)   r   r   c                 D    t        d   | j                  |       d      S )NSoftMarginLossr}   r~   r   r   s     r'   r>   z/softmarginloss_no_reduce_test.<locals>.<lambda>  s"    /0AIIaLFS r)   TFr   r5  r   s   @r'   softmarginloss_no_reduce_testrG  ~  sY    Aq-A+#KM]*(q1T ll$ $r)   c                       t        j                  dd      j                  d      j                          t	        dt         fd      dd d d	 fd
ddt         j                  	      S )NrR   r/   rM   "MultiLabelSoftMarginLoss_no_reducec                 R    t        j                  | j                  |       d      S r|   r   multilabel_soft_margin_lossr   r   s    r'   r>   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  s    a33Aqyy|vV r)   zF::multilabel_soft_margin_loss(
            i, t.to(i.options()), F::MultilabelSoftMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S r  r/  rl   r)   r'   r>   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  r   r)   r   r   c                     | j                         j                         z  dz
  |  j                         j                         z  z    j                  d      | j                  d      z  S Nr3   dimr   r   sumrd   r   s     r'   r>   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  s`    199;??$$A1"1C1C1E'EEFKKPQKRUVU[U[\]U^^ r)   Fr   r5   r   r   rq  r   rx   rc   r   s   @r'   'multilabelsoftmarginloss_no_reduce_testrU    sg    

1ba &&(A5#VXg+(q1_ll$ $r)   c                      t        j                  dd      j                  d      j                          t        j                  d      t	        dt         fd      dd d d	 fd
dddt         j                  
      S )NrR   r/   rM   *MultiLabelSoftMarginLoss_weights_no_reducec                 r    t        j                  | j                  |       j                  |       d      S r   rK  r   s    r'   r>   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  s.    a33Aqyy|;B??1;MY_a r)   zF::multilabel_soft_margin_loss(
            i, t.to(i.options()),
            F::MultilabelSoftMarginLossFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  .    t        j                  dd      S r  r/  rl   r)   r'   r>   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  r   r)   r   r   c                     | j                         j                         z  dz
  |  j                         j                         z  z    z  j                  d      | j                  d      z  S rO  rR  r   s     r'   r>   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  se    199;??$$A1"1C1C1E'EEFPUUZ[U\_`_e_efg_hh r)   TFr  rT  r   s   @@r'   /multilabelsoftmarginloss_weights_no_reduce_testr[    s~    

1ba &&(AjjnG=#abq ,(qWEi ll$ $r)   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrR   r0   MultiMarginLoss_no_reducec                 n    t        j                  | j                  |       j                         d      S r|   r   multi_margin_lossr   rb  r   s    r'   r>   z0multimarginloss_no_reduce_test.<locals>.<lambda>  '    a))!QYYq\->->-@FS r)   F::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultiMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S r  r/  rl   r)   r'   r>   z0multimarginloss_no_reduce_test.<locals>.<lambda>  r   r)   r   r   c                 t    t        d   | j                  j                  |       j                         d      S NMultiMarginLossr}   r~   r  r   s     r'   r>   z0multimarginloss_no_reduce_test.<locals>.<lambda>  1    01!QVV^^A5F5K5K5MY_` r)   TFr  r5   r   r   rq  rb  r   rx   rc   r   s   @r'   multimarginloss_no_reduce_testri    sp    

1!""$))+A,#SUo+(q1a ll$ $r)   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )Nr3   r0   MultiMarginLoss_1d_no_reducec                 n    t        j                  | j                  |       j                         d      S r|   r_  r   s    r'   r>   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  ra  r)   rb  c                  ,    t        j                  d      S r   r/  rl   r)   r'   r>   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  rB  r)   r   r   c                 t    t        d   | j                  j                  |       j                         d      S re  r  r   s     r'   r>   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  rg  r)   TFr  rh  r   s   @r'   !multimarginloss_1d_no_reduce_testro    sp    

1!""$))+A/#SUo((q1a ll$ $r)   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )Nrl   r0   ,multimarginloss_1d_input_0d_target_no_reducec                 n    t        j                  | j                  |       j                         d      S r|   r_  r   s    r'   r>   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  ra  r)   rb  c                  ,    t        j                  d      S r   r/  rl   r)   r'   r>   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  rB  r)   r   r   c                 t    t        d   | j                  j                  |       j                         d      S re  r  r   s     r'   r>   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  rg  r)   TFr  rh  r   s   @r'   1multimarginloss_1d_input_0d_target_no_reduce_testru    sp    

21##%**,A?#SUo((q1a ll$ $r)   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrR   r0   MultiMarginLoss_p_no_reducec                 p    t        j                  | j                  |       j                         dd      S )NrM   r}   r<   r   r_  r   s    r'   r>   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s*    a))!QYYq\->->-@AQWX r)   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultiMarginLossFuncOptions().p(2).reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S )NrR   r/   r  gGz?)r5   r   r   rl   r)   r'   r>   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s    Q+224B r)   r   r   c                 v    t        d   | j                  j                  |       j                         dd      S )Nrf  rM   r}   ry  r  r   s     r'   r>   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s4    01!QVV^^A5F5K5K5MQR^de r)   TFr  rh  r   s   @r'    multimarginloss_p_no_reduce_testr|    sp    

1!""$))+A.#XZtB(q1f ll$ $r)   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrR   r0    MultiMarginLoss_margin_no_reducec                 p    t        j                  | j                  |       j                         dd      S r:  r_  r   s    r'   r>   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>  s*    a))!QYYq\->->-@X^_ r)   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::MultiMarginLossFuncOptions().margin(0.5).reduction(torch::kNone))c                  .    t        j                  dd      S r  r/  rl   r)   r'   r>   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>  r   r)   r   r   c                 v    t        d   | j                  j                  |       j                         dd      S )Nrf  r  r}   r;  r  r   s     r'   r>   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>  s6    01!QVV^^A5F5K5K5M9<P r)   TFr  rh  r   s   @r'   %multimarginloss_margin_no_reduce_testr    st    

1!""$))+A3#_aT ,(q1P !ll$ $r)   c                  B    t        j                  d      j                  d      j                         j	                          t        j                  dt         j
                        t        dt         fd      dd d	 d
 fddddt         j
                  
      S )NrR   r0   r/   r   !MultiMarginLoss_weights_no_reducec                     t        j                  | j                  |       j                         j                  |       d      S r   r_  r   s    r'   r>   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>  s6    a))!QYYq\->->-@YZI[4:< r)   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::MultiMarginLossFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  .    t        j                  dd      S r  r/  rl   r)   r'   r>   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>  r   r)   r   r   c                 v    t        d   | j                  j                  |       j                         d      S )Nrf  r}   r   r  r   s     r'   r>   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>  s6    01!QVV^^A5F5K5K5M9@FT r)   TFr  )r5   r   r   rq  rb  rc   r   rx   r   s   @@r'   &multimarginloss_weights_no_reduce_testr    s    

1!""$))+Ajj5<<0G4#<=h ,(qWET !ll!$ $r)   c                     d } ||       }t        |t        j                        r|gn|}t               5   || j	                  d      cddd       S # 1 sw Y   yxY w)zReference function for modules supporting no batch dimensions.

    The module is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    c                     t        | t        t        f      r| D cg c]  }|j                  d       c}S | j                  d      S c c}w rJ   
isinstancelisttuple	unsqueezeinpr7   s     r'   unsqueeze_inpz0single_batch_reference_fn.<locals>.unsqueeze_inp-  =    cD%=),/0qAKKN00}}Q 1   Ar   N)r  r5   Tensorr   squeeze)r  
parametersmoduler  single_batch_inputs        r'   single_batch_reference_fnr  '  sa     
 'u-1;<NPUP\P\1],-cu		 6)*22156 6 6s   AAc                  <   g t               t               t               t               t	               t               t               t               t               t               t               t               t               t               t               t               t!               t#               t%               t'               t)               t+               t-               t/               t1               t3               t5               t7               t9               t;               t=               t?               tA               tC               tE               tG               tI               tK               tM               tO               tQ               tS               tU               tW               tY               t[               t]               t_               ta               tc               te        dddddddtf        jh                        te        ddd	ddd
ddtf        jh                  	      te        ddddddddtf        jh                  	      te        ddddddddtf        jh                  	      te        ddddddddtf        jh                  	      te        ddddddddtf        jh                  	      te        dddddddd      te        dd ddddtf        jh                         te        d!d" d#d$dddtf        jh                  %      te        d&d' d(ddddtf        jh                  %      te        d)d* d+ddddtf        jh                  %      te        d,d- d.ddddtf        jh                  %      te        d/d0 d1ddddtf        jh                  %      te        d2d3 d4dd5ddtf        jh                  6      te        d2d7d8d9dd:ddtf        jh                  	      te        d2d;d<d9dd=ddtf        jh                  	      te        d>d? d@ddAddtf        jh                  6      te        dBdCdDdEddddtf        jh                  F	      te        dBdGdHdIddJdddtf        jh                  K
      te        dBdLdMdIddNdddtf        jh                  K
      te        dBdOdPdQdd=dddtf        jh                  K
      te        dBdRdSdTdd:dddUtf        jh                  K
      te        dBdCdDdVddddW      te        dXdY dZd[ddddUtf        jh                  \	      te        d]d^ dZd[dddUtf        jh                  _      te        d`da dbdcdddtf        jh                  %      te        ddde dfdcdddtf        jh                  %      te        dgdh didcdddtf        jh                  %      te        djdkdlddmdddtf        jh                  n	      te        djdodpdqdd=dddtf        jh                  K
      te        djdrdsdqdd:dddtf        jh                  K
      te        dtdu dvdwddddtf        jh                  \	      te        dxdy dzd{ddtf        jh                         te        d|d} d~d{ddtf        jh                         te        dd dd{ddtf        jh                         te        dd dd{ddtf        jh                         te        dd ddddtf        jh                         te        ddddddddtf        jh                  F	      te        dddddd:dddtf        jh                  K
      te        dddddddddtf        jh                  K
      te        dddddd
dddtf        jh                  K
      te        dddddddddtf        jh                  K
      te        dddddddd      te        dd ddddddtf        jh                  \	      te        dd ddddtf        jh                         te        dd ddddtf        jh                         te        dd dddddtf        jh                  %      te        dd dddddtf        jh                  %      te        dd dddddtf        jh                  %      te        dddddddtf        jh                        te        dddddd=ddtf        jh                  	      te        ddddtf        jh                        te        ddddtj        dtf        jh                        te        dddd dd      te        ddddń dtf        jh                  tm        dƫ      ǫ      te        ddddȄ ddtf        jh                  tm        dƫ      ʫ      te        dddd̈́ ddtf        jh                  ϫ      te        ddddЄ ddtf        jh                  ϫ      te        ddddӄ ddtf        jh                  ϫ      te        ddddׄ ddtf        jh                  ϫ      te        ddڄ dd܄ dtf        jh                  ݫ      te        dd߄ dd dtf        jh                  ݫ      te        dd dd dtf        jh                  ݫ      te        dd dd dd      te        d dd ddd      te        ddddtf        jh                        te        ddddtf        jh                        te        to        tp        jr                  dd d      ddddtf        jh                        te        to        tp        jr                  dd d      dddd       te        to        tp        jr                  dd d      ddddtf        jh                        te        to        tp        jr                  d dd      ddddtf        jh                        te        to        tp        jr                  dd dd      d	dd
dtf        jh                        te        to        tp        jr                  dd dd      ddddtf        jh                        te        to        tp        jr                  d ddd      ddddtf        jh                        te        to        tp        jr                  dd dd      d	ddd       te        to        tp        jr                  dd dd      ddddtf        jh                        te        to        tp        jr                  d ddd      ddddtf        jh                        te        to        tp        jr                  dd d      ddddtf        jh                        te        to        tp        jr                  dd d      ddddtf        jh                        te        to        tp        jr                  dd d      ddddtf        jh                        te        to        tp        jr                  d dd      d dd!dtf        jh                        te        to        tp        jr                  dd d      dd"d#d       te        to        tp        jr                  dd d$d      d%dd&dtf        jh                        te        to        tp        jr                  dd d$d      d%d"d'd       te        to        tp        jr                  d(d d$d      d)d*d+dtf        jh                        te        to        tp        jr                  d dd$d      d,dd-dtf        jh                        te        to        tp        jr                  d d.d$d      d/dd0dtf        jh                        te        to        tp        jr                  d d1d$d      d2dd3dtf        jh                        te        to        tp        jr                  d(d d$d      d4dd5dtf        jh                        te        to        tp        jr                  d d1d$d      d6dd7dtf        jh                        te        to        tp        jr                  dd d8d      d9dd:dtf        jh                        te        to        tp        jr                  dd d8d      d9d"d;d       te        to        tp        jr                  d(d d8d      d<d*d=dtf        jh                        te        to        tp        jr                  d dd8d      d>dd?dtf        jh                        te        to        tp        jr                  d d.d8d      d@ddAdtf        jh                        te        to        tp        jr                  d d1d8d      dBddCdtf        jh                        te        to        tp        jr                  d(d d8d      dDddEdtf        jh                        te        to        tp        jr                  d d1d8d      dFddGdtf        jh                        te        to        tp        jr                  dd d      dHdIdJdtf        jh                        te        to        tp        jr                  dd d      dHdKdLd       te        to        tp        jr                  dMd d      dNdOdPdtf        jh                        te        to        tp        jr                  d dd      dQdIdRdtf        jh                        te        to        tp        jr                  dd dSd      dTdIdUdtf        jh                        te        to        tp        jr                  dd dSd      dTdKdVd       te        to        tp        jr                  dWd dSd      dXdYdZdtf        jh                        te        to        tp        jr                  d d[dSd      d\dd]d^dtf        jh                  _      te        to        tp        jr                  dWd dSd      d`dYdadtf        jh                        te        to        tp        jr                  d d[dSd      dbdOdcd^dtf        jh                  _      te        to        tp        jt                  dde      dfdgdhdtf        jh                        te        to        tp        jt                  ditf        jv                  j      dkdgdlddtf        jh                  m      te        to        tp        jt                  die      dndodpdtf        jh                        te        to        tp        jt                  die      dndqdrdtf        jh                        te        to        tp        jt                  ditf        jv                  j      dkdqdsddtf        jh                  m      te        to        tp        jt                  dte      dudvdwddtf        jh                  x      te        to        tp        jt                  dye      dzdvd{ddtf        jh                  x      te        to        tp        jt                  dde      dfd|d}dd~      te        to        tp        jx                  dde      ddgddtf        jh                        te        to        tp        jx                  die      ddoddtf        jh                        te        to        tp        jx                  die      ddqddtf        jh                        te        to        tp        jx                  dte      ddvddtf        jh                        te        to        tp        jx                  dye      ddvddtf        jh                        te        to        tp        jx                  dte      dd|dd       te        dd ddddtf        jh                        te        dd ddddtf        jh                        te        dd dddtj        dtf        jh                        te        dd ddddtf        jh                        te        dd ddddtf        jh                        te        dd ddtj        ddtf        jh                        te        dddd|ddtf        jh                        te        dd tf        jh                        te        dd dtf        jh                        te        dd dtf        jh                        te        dddd dtf        jh                        te        dd tj        dtf        jh                        te        dddddddddtf        jh                  
      te        dddddtp        jz                  fdddddt|        rdndtf        jh                  	      te        dddÐdĄ ddddtf        jh                  ū	      te        dddddtp        jz                  fdƐdǄ ddddtf        jh                  ū	      te        dȐddddddtp        j~                  fdɐdʄ dddt|        rdndtf        jh                  ū	      te        d͐dΐdϐdЄ dф dddtf        jh                  ҫ	      te        dӐdԐdՐdtj        dtf        jh                  ׫      te        dؐdِdtg        j                  ddg      fdtj        dtf        jh                  ׫      te        dg dܢddfdސdddddd	      } t        g dg d      D ]  \  }}dD ]  }|dyk(  r|dk(  rt        t        di|diz               }ddj                  t        t        |            z   dz   }dd|z  z   }dt        d |D              z   }| j                  te        d| dddydyd|didid|f	d| d| d| d||d| dddtf        jh                  
               g d}i ddtf        jh                  dddd1idddtf        jh                  ddddtf        jh                  dddtf        jh                  dddtf        jh                  iddtf        jh                  id dtf        jh                  iddtf        jh                  iddtf        jh                  iddtf        jh                  iddtf        jh                  iddtf        jh                  iddtf        jh                  iddtf        jh                  iddtf        jh                  id	dtf        jh                  idtf        jh                  idtf        jh                  idtf        jh                  idtf        jh                  idtf        jh                  idtf        jh                  idtf        jh                  id
}	|D ]L  }
te        |
dtj        dd      }|	j                  |
i       }|j                  |       | j                  |       N | S (  NConv1d)r1   rR   rW   z!torch::nn::Conv1dOptions(4, 5, 3))rM   r1   r/   Tr?   )r@   rA   rB   rC   cudnnrE   rF   rG   )r1   rR   rW   rM   z+torch::nn::Conv1dOptions(4, 5, 3).stride(2)stride)	r@   rA   rB   rC   r  rK   rE   rF   rG   )r1   rR   rW   r3   r3   z6torch::nn::Conv1dOptions(4, 5, 3).stride(1).padding(1)pad1r  )r1   rR   rR   r3   rM   z6torch::nn::Conv1dOptions(4, 5, 5).stride(1).padding(2)pad2)r1   r1   rW   r3   r3   z6torch::nn::Conv1dOptions(4, 4, 3).stride(1).padding(1))r3   r1   r3   	pad1size1)r1   r1   rR   r3   rM   z6torch::nn::Conv1dOptions(4, 4, 5).stride(1).padding(2)	pad2size1)r   r1   r/   
zero_batch)r@   rA   rB   rC   r  rK   rE   rF   Conv1d_dilatedc                  4    t        j                  dddd      S )Nr1   rR   rW   rM   kernel_sizedilationrv   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>      		!QA J r)   z-torch::nn::Conv1dOptions(4, 5, 3).dilation(2))r   r   rB   rC   rE   rF   rG   Conv1d_groupsc                  4    t        j                  dddd      S )Nr1   r^   rW   rM   r  groupsr  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>      		!QAa H r)   z+torch::nn::Conv1dOptions(4, 6, 3).groups(2))rM   r1   r^   )r   r   rB   rC   r  rE   rF   rG   Conv1d_pad_validc                  4    t        j                  dddd      S )Nr1   rR   rW   validpaddingr  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    		!Q7 C r)   z8torch::nn::Conv1dOptions(4, 5, 3).padding(torch::kValid)Conv1d_pad_samec                  4    t        j                  dddd      S )Nr1   rR   rW   samer  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>      		!Q6 B r)   z7torch::nn::Conv1dOptions(4, 5, 3).padding(torch::kSame)Conv1d_pad_same2c                  4    t        j                  dddd      S )Nr1   rR   r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r  r)   z7torch::nn::Conv1dOptions(4, 5, 4).padding(torch::kSame)Conv1d_pad_same_dilatedc                  6    t        j                  ddddd      S )Nr1   rR   r  rM   r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    		!Q6A N r)   zCtorch::nn::Conv1dOptions(4, 5, 3).padding(torch::kSame).dilation(2)ConvTranspose1dc                  8    t        j                  dddddd      S )NrW   r1   rW   r3   r3   )r  r  r  output_paddingrv   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s     2 21aQt]^os t r)   zQtorch::nn::ConvTranspose1dOptions(3, 4, 3).stride(3).padding(1).output_padding(1))r3   rW      )r   r   rB   r  rC   rE   rF   rG   )rW   r1   rW   rM   r3   r3   r3   Fztorch::nn::ConvTranspose1dOptions(3, 4, 3)
                                    .stride(2).padding(1).output_padding(1).groups(1).bias(false))r3   rW   r^   rH   )	rW   r1   rW   rM   r3   r3   r3   TrM   ztorch::nn::ConvTranspose1dOptions(3, 4, 3)
                                    .stride(2).padding(1).output_padding(1).groups(1).bias(true).dilation(2)dilatedConvTranspose1d_groupsc            	      :    t        j                  ddddddd      S )	Nr1   r^   rW   r  r3   r  rM   )r  r  r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s      2 21a4QRcgpq r r)   ztorch::nn::ConvTranspose1dOptions(4, 6, 3)
                                    .stride(3).padding(1).output_padding(1).groups(2))rM   r1   r  Conv2d)rW   r1   rW   rM   z&torch::nn::Conv2dOptions(3, 4, {3, 2}))rM   rW   r  rR   )	r@   rA   rB   rC   r  check_with_long_tensorrE   rF   rG   )rW   r1   rW   rW   rM   rM   z5torch::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2})r]   strided)
r@   rA   rB   rC   r  rK   r  rE   rF   rG   )rW   r1   r  r  r3   r3   zEtorch::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2}).padding({1, 1})r  )rW   rM   r  r  r  r  zVtorch::nn::Conv2dOptions(3, 2, {3, 3}).stride({2, 2}).padding({1, 1}).dilation({2, 2}))rM   rW   r0   r0   )rW   r1   r  r3   r   r3   r3   Fztorch::nn::Conv2dOptions(3, 4, {3, 2})
                                    .stride(1).padding(0).dilation(1).groups(1).bias(false))rM   rW   r^   rR   gQ?)r   rW   r  rR   )r@   rA   rB   rC   r  rK   r  rE   Conv2d_groupsc                  4    t        j                  dddd      S Nr1   r^   r  rM   r  rv   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>g      		!Qq A r)   z0torch::nn::Conv2dOptions(4, 6, {3, 2}).groups(2))rM   r1   r^   rR   )	r   r   rB   rC   r  r  rE   rF   rG   Conv2d_groups_thnnc                  4    t        j                  dddd      S r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>r  r  r)   )r   r   rB   rC   r  rE   rF   rG   Conv2d_pad_validc                  4    t        j                  dddd      S )NrM   r1   rW   r1   r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>|  s    		!Q H r)   z=torch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kValid))rM   rM   r^   rR   Conv2d_pad_samec                  4    t        j                  dddd      S )NrM   r1   r  r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    		!Q G r)   z<torch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kSame)Conv2d_pad_same_dilatedc                  6    t        j                  ddddd      S )NrM   r1   r  r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    		!QQR S r)   zHtorch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kSame).dilation(2)ConvTranspose2d)rW   r1   rW   r  r3   r  ztorch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({3, 2}).padding(1).output_padding({1, 1}))r3   rW   r  r^   )	r@   rA   rB   r  rC   r  rE   rF   rG   )	rW   r1   rW   rM   rW   r3   r  r3   Fr  a`  torch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({2, 3})
                                    .padding(1)
                                    .output_padding({1, 1})
                                    .groups(1)
                                    .bias(false)
                                    .dilation({2, 2}))r3   rW   r^   r  )rW   r1   rW   r  r3   r  r3   Fztorch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({2, 3}).padding(1).output_padding({1, 1}).groups(1).bias(false)ConvTranspose2d_groupsc                  4    t        j                  dddd      S )NrM   r1   r  r  )rv   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s     2 21a J r)   z9torch::nn::ConvTranspose2dOptions(2, 4, {2, 3}).groups(2))r3   rM   r1   rR   Conv2d_depthwisec                  4    t        j                  dddd      S )Nr1   r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r  r)   z0torch::nn::Conv2dOptions(4, 4, {3, 3}).groups(4))rM   r1   r^   r^    Conv2d_depthwise_with_multiplierc                  4    t        j                  dddd      S )Nr1   r0   r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r  r)   z0torch::nn::Conv2dOptions(4, 8, {3, 3}).groups(4)Conv2d_depthwise_stridedc                  6    t        j                  ddddd      S )Nr1   r  r  )r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    		!Qva P r)   z?torch::nn::Conv2dOptions(4, 4, {3, 3}).stride({2, 2}).groups(4)Conv2d_depthwise_paddedc                  6    t        j                  ddddd      S )Nr1   r  r  )r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    		!Qq Q r)   z@torch::nn::Conv2dOptions(4, 4, {3, 3}).padding({1, 1}).groups(4)Conv2d_depthwise_dilatedc                  6    t        j                  ddddd      S )Nr1   r  )r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    		!QPQ R r)   zAtorch::nn::Conv2dOptions(4, 4, {2, 2}).dilation({2, 2}).groups(4))rM   r1   rR   rR   Conv3d)rM   rW   rM   rW   rM   z)torch::nn::Conv3dOptions(2, 3, {2, 3, 2}))r3   rM   r1   rR   r1   g?)rM   rW   rM   rW   r1   r3   r   r3   r3   Fztorch::nn::Conv3dOptions(2, 3, {2, 3, 4})
                                    .stride(1).padding(0).dilation(1).groups(1).bias(false))r3   rM   rW   r1   rR   )rM   rW   )r3   r3   r3   r3   r   r3   r3   F1x1x1_no_biasF)rW   r1   rM   rM   z+torch::nn::Conv3dOptions(3, 4, 2).stride(2))rM   rW   rR   rR   rR   )rW   r1   rM   rM   r3   z6torch::nn::Conv3dOptions(3, 4, 2).stride(2).padding(1)stride_padding)rW   r1   r  z)torch::nn::Conv3dOptions(3, 4, {2, 3, 4}))r   rW   rW   r1   rR   )r@   rA   rB   rC   r  r  rK   rE   Conv3d_groupsc                  4    t        j                  dddd      S )NrM   r1   rW   r  rv   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>D  r  r)   z+torch::nn::Conv3dOptions(2, 4, 3).groups(2)Conv3d_dilatedc                  4    t        j                  dddd      S )NrW   r1   rM   r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>O  r  r)   z-torch::nn::Conv3dOptions(3, 4, 2).dilation(2)Conv3d_dilated_stridedc                  6    t        j                  ddddd      S )NrW   r1   rM   )r  r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>X  s    		!QARS T r)   z7torch::nn::Conv3dOptions(3, 4, 2).dilation(2).stride(2)Conv3d_pad_validc                  4    t        j                  dddd      S )NrW   r1   r  r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>a  s    		!Q	7 K r)   z@torch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kValid))rM   rW   r^   rR   r1   Conv3d_pad_samec                  4    t        j                  dddd      S )NrW   r1   r  r  r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>k  s    		!Q	6 J r)   z?torch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kSame)Conv3d_pad_same_dilatedc                  6    t        j                  ddddd      S )NrW   r1   r  r  rM   r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>u  s    		!Q	6TU V r)   zKtorch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kSame).dilation(2)ConvTranspose3dz2torch::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2}))r@   rA   rB   r  rC   rE   rF   rG   )	rM   rW   r  r3   r   r   r3   T)rM   rM   rM   ztorch::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2})
                                    .stride(1).padding(0).output_padding(0).groups(1).bias(true).dilation({2, 2, 2}))	r@   rA   rB   r  rC   rK   rE   rF   rG   ReplicationPad3d))r3   rM   rW   rW   rM   r3   z6torch::nn::ReplicationPad3dOptions({1, 2, 3, 3, 2, 1}))rM   rW   rM   rM   rM   )r@   rA   rB   rC   rG   )rW   rM   rM   rM   no_batch_dim)r@   rA   rB   rC   rD   rK   rG   c            	      V    t        j                  dddddt         j                  d      S )NrM   rW   T)r   requires_grad)r5   r   
complex128rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s"    UZZ1aAU=M=M]ab r)   complex)r@   rA   rB   r   	skip_halfrK   	Embedding)r1   rW   z!torch::nn::EmbeddingOptions(4, 3)c                  l    t        j                  ddt         j                        j                  d      S NrM   rW   r   r1   r5   ro  rb  random_rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  "    U[[AUZZ@HHK r)   z0https://github.com/pytorch/pytorch/issues/117971)r@   rA   rB   r   r_   rG   	decoratorc                      t        j                  ddt         j                        j                  d      j	                  dd      S Nr3   i   r   r1   r  r5   ro  rb  r
  r9   rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  1    U[[CuzzBJJ1MTTUVX[\ r)   discontiguous)r@   rA   rB   r   r_   rK   rG   r  EmbeddingBagz$torch::nn::EmbeddingBagOptions(4, 3)c                  l    t        j                  ddt         j                        j                  d      S r  r	  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r  r)   mean)r@   rA   rB   r   r_   rK   rG   c                      t        j                  ddt         j                        j                  d      j	                  dd      S r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r  r)   )r1   rW   N       @FrS  ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kSum)c                  l    t        j                  ddt         j                        j                  d      S r  r	  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r  r)   rS  )r1   rW   Nr  Fmaxztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kMax)c                  l    t        j                  ddt         j                        j                  d      S r  r	  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r  r)   r  EmbeddingBag_mean_padding_idxc                  2    t        j                  ddd      S )Nr1   rW   r3   padding_idxrv   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    1! D r)   z3torch::nn::EmbeddingBagOptions(4, 3).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S NrW   r5   stackrb   rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  &    U[[%..*;U^^A=N)OP r)   )r   r   rB   r   r_   rG   EmbeddingBag_sum_padding_idxc            	      :    t        j                  ddd dddd      S )Nr1   rW   r  FrS  r3   r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>      1dBuZ[ \ r)   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kSum).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S r   r!  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r#  r)   EmbeddingBag_max_padding_idxc            	      :    t        j                  ddd dddd      S )Nr1   rW   r  Fr  r3   r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r&  r)   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kMax).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S r   r!  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  r#  r)   EmbeddingBag_sparsec                  P    t        j                  dddt        j                        S )Nr1   rW   T)sparser   )rv   r  r5   rc   rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>   s    1T V r)   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .sparse(true)._weight(torch::rand({4, 3}).to(torch::kFloat64))c                  L    t        j                  d      j                  dd      S NrM   r3   r5   rb   repeatrl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>      U^^A.55a; r)   )r   r   rB   r   r_   has_sparse_gradientsc                  P    t        j                  ddt        j                  d      S )Nr1   rW   T)r   r-  )rv   r  r5   rc   rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>  s    Qd S r)   z_torch::nn::EmbeddingOptions(4, 3).sparse(true)._weight(torch::rand({4, 3}).to(torch::kFloat64))c                  L    t        j                  d      j                  dd      S r/  r0  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>
  r2  r)   Embedding_sparse)r   rB   r   r   r_   r3  PixelShuffler  z!torch::nn::PixelShuffleOptions(3))r3   	   r1   r1   PixelUnshufflez#torch::nn::PixelUnshuffleOptions(3))r3   r3      r:  r:  nearest)rd   scale_factormodezF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12})).scale_factor(std::nullopt).mode(torch::kNearest))r3   rM   r1   interpolate_nearest_1d)r   cpp_options_argsrC   r   r   rG   )r   rM   r1   interpolate_nearest_1d_zero_dim)r   r?  rC   r   r   )r:  )r3   rM   rW   interpolate_nearest_tuple_1dg      @zF::InterpolateFuncOptions()
                                .size(std::nullopt).scale_factor(std::vector<double>({4.})).mode(torch::kNearest)interpolate_nearest_scale_1dlinear)rd   r<  r=  align_cornersa  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(false)interpolate_linear_1d)r1   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(false)interpolate_linear_tuple_1da  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4.}))
                                .mode(torch::kLinear)
                                .align_corners(false)interpolate_linear_scale_1dinterpolate_linear_1d_zero_dima  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(true)#interpolate_linear_1d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4.}))
                                .mode(torch::kLinear)
                                .align_corners(true))interpolate_linear_scale_1d_align_cornersrM   zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({2, 2}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r3      r3   r3   %interpolate_nearest_2d_launch_configszF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r3   rM   r1   r1   interpolate_nearest_2d)r:     zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 16}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r3   rM   rW   r1   interpolate_nearest_tuple_2dzF::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kNearest)interpolate_nearest_scale_2d)r   rM   r1   r1   interpolate_nearest_2d_zero_dimbilineara  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(false)interpolate_bilinear_2d interpolate_bilinear_2d_zero_dim)r1   r^   a	  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(false))r3   rM   rM   rW   interpolate_bilinear_tuple_2da
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)interpolate_bilinear_scale_2d)r  r  a
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 2.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)*interpolate_bilinear_scale_tuple_shared_2d)r        ?a
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)*interpolate_bilinear_scale_tuple_skewed_2da  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(true)+interpolate_bilinear_tuple_2d_align_cornersa	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBilinear)
                                .align_corners(true)8interpolate_bilinear_scale_tuple_skewed_2d_align_cornersbicubica
  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(false)interpolate_bicubic_2dinterpolate_bicubic_2d_zero_dima  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(false)interpolate_bicubic_tuple_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kBicubic)
                                .align_corners(false)interpolate_bicubic_scale_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 2.}))
                                .mode(torch::kBicubic)
                                .align_corners(false))interpolate_bicubic_scale_tuple_shared_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBicubic)
                                .align_corners(false))interpolate_bicubic_scale_tuple_skewed_2da  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(true)*interpolate_bicubic_tuple_2d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBicubic)
                                .align_corners(true)7interpolate_bicubic_scale_tuple_skewed_2d_align_cornerszF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r3   rM   r1   r1   r1   interpolate_nearest_3d)r   rM   r1   r1   r1   interpolate_nearest_3d_zero_dim)r:  rN  rN  zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 16, 16}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r3   rM   rW   r1   r1   interpolate_nearest_tuple_3dzF::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4., 4.}))
                                .mode(torch::kNearest)interpolate_nearest_scale_3d	trilineara  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(false)interpolate_trilinear_3d!interpolate_trilinear_3d_zero_dim)r1   r^   r^   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(false))r3   rM   rM   rW   rW   interpolate_trilinear_tuple_3dg      @a  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({3., 3., 3.}))
                                .mode(torch::kTrilinear)
                                .align_corners(false)interpolate_trilinear_scale_3dr   )r   r?  rC   r   r   r   rG   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(true),interpolate_trilinear_tuple_3d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({3., 3., 3.}))
                                .mode(torch::kTrilinear)
                                .align_corners(true),interpolate_trilinear_scale_3d_align_cornersr4   rP  zF::SoftmaxFuncOptions(-1))rM   rK  softmax_lastdimr3   )rQ  r   z/F::SoftmaxFuncOptions(1).dtype(torch::kFloat64)softmax_lastdim_dtype)r   r?  rC   r   r   rN   rG   zF::SoftmaxFuncOptions(1))rM   rK  rM   rM   softmax_spatial_special)rM   rM   r1   r1   softmax_spatialsoftmax_spatial_dtyper   zF::SoftmaxFuncOptions(0)rV   softmax_functional_dim0)r   r?  rC   r   rN   r   rG   rW   zF::SoftmaxFuncOptions(3)softmax_functional_dim3rl   softmax_functional_scalar)r   r?  rC   r   rN   r   zF::LogSoftmaxFuncOptions(-1)log_softmax_lastdimzF::LogSoftmaxFuncOptions(1)log_softmax_spatial_speciallog_softmax_spatialzF::LogSoftmaxFuncOptions(0)log_softmax_dim0zF::LogSoftmaxFuncOptions(3)log_softmax_dim3log_softmax_scalarUnfoldc                  2    t        j                  dddd      S )Nr  r  r   r   rv   r~  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>q	  s    		&&&& I r)   zPtorch::nn::UnfoldOptions({2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1}))rM   r1   rW   rW   )r   r   rB   rC   r_   rN   rG   Foldc                  4    t        j                  ddddd      S Nr  r  r  r  rv   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>z	       O r)   zVtorch::nn::FoldOptions({3, 3}, {2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1}))rM   rN  r1   Fold_no_batch_dim_inputc                  4    t        j                  ddddd      S r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  r  r)   )rN  r1   )r   r   rB   rC   r_   refrN   rG   Unfold_int_inputc                  2    t        j                  dddd      S )NrM   r3   r   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  s    		!Q1 5 r)   z<torch::nn::UnfoldOptions(2).dilation(1).padding(0).stride(1)Fold_int_inputc                  4    t        j                  ddddd      S NrW   rM   r3   r   r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	      1aA 6 r)   z=torch::nn::FoldOptions(3, 2).dilation(1).padding(0).stride(1)Fold_no_batch_dim_int_inputc                  4    t        j                  ddddd      S r  r  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  r  r)   )r   r   rB   rC   r  r_   rN   rG   rL   rO   rQ   with_up_down_scalarrT   PairwiseDistancec                  Z    t        j                  dd      t        j                  dd      fS Nr/   r0   r/  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  !    ekk"a0%++b!2DE r)   )r@   r   rG   c                  Z    t        j                  dd      t        j                  dd      fS )Nr/   r3   r0   r/  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  r  r)   broadcast_lhs)r@   r   rK   rG   c                  Z    t        j                  dd      t        j                  dd      fS )Nr/   r0   r3   r/  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  s!    ekk"a0%++a2CD r)   broadcast_rhs)g      ?r   TzDtorch::nn::PairwiseDistanceOptions().p(1.5).eps(1e-05).keepdim(true)c                  Z    t        j                  dd      t        j                  dd      fS r  r/  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  r  r)   with_non_default_args)r@   rA   rB   r   rK   rG   c                  V    t        j                  d      t        j                  d      fS )Nr0   r/  rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  s    ekk!nekk!n= r)   )r@   r   rD   rK   rG   TransformerEncoderLayer)r1   rM   rN          ztorch::nn::TransformerEncoderLayerOptions(4, 2)
                                    .dim_feedforward(16)
                                    .dropout(0.0)r  relu_activationrP   )
r@   rA   rB   rC   rK   rE   rF   r`   r_   rG   r1   r0   r  ztorch::nn::TransformerEncoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kGELU)gelu_activationg{Gz?)	r@   rA   rB   rC   r_   rK   rE   rF   rG   TransformerDecoderLayer)r1   rM   r0   r  ztorch::nn::TransformerDecoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)c                  ^    t        j                  ddd      t        j                  ddd      fS NrW   r1   rM   r   rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>	  %    ejjAq15::aA3FG r)   )	r@   rA   rB   r   r_   rK   rE   rF   rG   ztorch::nn::TransformerDecoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kGELU)c                  ^    t        j                  ddd      t        j                  ddd      fS r  r   rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>
  r  r)   Transformera  torch::nn::TransformerOptions()
                                    .d_model(4)
                                    .nhead(2)
                                    .num_encoder_layers(2)
                                    .num_decoder_layers(2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kReLU)c                      t        j                  ddd      t        j                  ddd      t        j                  dd      fS r  r   rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda>
  s4    ejjAq15::aA3F

STVWHXY r)   multilayer_codergQ?r.   )rW   rR   ztorch::nn::LinearOptions(3, 5)c                  ,    t        j                  d      S r   r   rl   r)   r'   r>   z&get_new_module_tests.<locals>.<lambda> 
  s    UZZ] r)   c                     t        j                  | j                  dd      |d   j                               j                  d      |d   z   S )Nr3   r4   r   )r5   r6   r8   r7   r:   s      r'   r>   z&get_new_module_tests.<locals>.<lambda>!
  s=    !&&B-1)J)O)OPR)SVWXYVZ)Z r)   )	r@   rA   rB   r   rD   rK   rE   rF   rG   rU   z5torch::nn::FlattenOptions().start_dim(-3).end_dim(-1))r4   )rW   r1   rR   )r@   rB   rA   rC   rD   rK   rG   	Unflattenz'torch::nn::UnflattenOptions(-2, {2, 2})	LayerNorm)8   r  r  r   zMtorch::nn::LayerNormOptions({56, 56, 56}).eps(1e-5).elementwise_affine(false))r1   r  r  r  3d_no_affine_large_feature)	r@   rA   rB   rC   r  
check_evalgradcheck_fast_mode
check_halfrK   )reflectcircular	replicater  )ztorch::kReflectztorch::kCircularztorch::kReplicateztorch::kZerosr  {z, }r  r  c              3   &   K   | ]	  }|d z     ywr3   Nrl   .0r<   s     r'   	<genexpr>z'get_new_module_tests.<locals>.<genexpr>V
  s     (@1Q(@s   Convdztorch::nn::Convz~dOptions(2, 3, 3)
                                            .stride(2)
                                            .padding(z)
                                            .dilation(1)
                                            .groups(1)
                                            .bias(true)
                                            .padding_mode()_stride2_pad2)
r@   rA   rB   rC   output_sizer  rK   rE   rF   rG   )ELU
HardshrinkHardsigmoidHardtanh	Hardswish	LeakyReLU
LogSigmoidPReLUReLUReLU6rL   SELUCELUGELUGLUr   SiLUMishSoftplus
SoftshrinkSoftsignTanh
Tanhshrink	Thresholdr  )r  )rA   rG   r  rA   r  )r_   	check_jitrG   r  )rN   rG   r  rG   r  r  r  r  r  r  r  r  r  r  r  )r  r   r  r  r  r  r  )r@   rC   rD   rK   test_cpp_api_parity)Jr   r   r   r   r   r   r   r   r   r  r  r  r  r%  r6  r=  rD  rV  r[  rs  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r#  r)  r6  r?  rG  rU  r[  ri  ro  ru  r|  r  r  r   r5   rc   r  r   rx   r   interpolatesoftmaxfloat64log_softmaxgelur   reluSizezipr  rangejoinmapr|  appendgetupdate)new_module_testspadding_modecpp_padding_moder  r  cpp_paddingrC   r  non_linear_activations_no_batch!non_linear_activations_extra_infonon_linear_activationactivation_test_info
extra_infos                r'   get_new_module_testsr  8  s&#   K%'K K 	'(K 	+,	K
 	)*K 	&'K 	./K 	01K 	-.K 	!"K 	()K 	12K 	,-K 	34K 	K  	&'!K" 	%&#K$ 	 %K& 	&''K( 	 )K* 	,-+K, 	'(-K. 	45/K0 	891K2 	!"3K4 	)*5K6 	./7K8 	!"9K: 	)*;K< 	./=K> 	$%?K@ 	+,AKB 	 CKD 	$%EKF 	GKH 	/0IKJ 	/0KKL 	,-MKN 	,-OKP 	*+QKR 	12SKT 	&'UKV 	01WKX 	89YKZ 	'([K\ 	*+]K^ 	:;_K` 	)*aKb 	./cKd 	/0eKf 	 &!D! ,,		
gKz 	 )!N! ,,
	
{KP 	 ,!Y!,,
	
QKf 	 ,!Y! ,,
	
gK| 	 ,!Y  ,,
	
}KR 	 ,!Y  ,,
	
SKh 	 &!D! 		
iK| 	%J!P! ,,	
}KN 	$H!N  ,,		
OKb 	'C![! ,,		
cKv 	&B!Z! ,,		
wKJ 	'B!Z! ,,		
KK^ 	.N!f! ,,		
_Kr 	&t!t  ,,		
sKF 	)9"e  ,,	
GK^ 	);"p  ,,	
_Kv 	-r"Y  ,,
	
wKL 	 +!I##' ,,
	
MKb 	 3!X##' ,,	
cKz 	 ;!h##' ,,	
{KR 	 C!y##' ,,	
SKj 	 >"_##' ,,	
kKD	 	 +!I##'		
E	KX	 	$A!S##' ,,
	
Y	Kn	 	)A!S##' ,,		
o	KB
 	'H!`# ,,		
C
KV
 	&G!_#,,		
W
Kj
 	.S!k#,,		
k
K~
 	)9"Y##',,	

KV 	)K"9 $#',,!	
WKz 	)C"o##',,	
{KT 	-J!\##',,
	
UKj 	'A!S# ,,	
kK| 	7A!S# ,,	
}KN 	/P!b# ,,	
OK` 	.Q!c# ,,	
aKr 	/R!d# ,,	
sKD 	 .!L&#',,
	
EKZ 	 A"_&#',,	
[Kt 	 A"_& #(,,	
uKN 	 )!N&#',,	
OKf 	 ,!Y&!#',,	
gK~ 	 .!L&#'		
KR 	$H!N&#' ,,
	
SKh 	%J!P&,,	
iKz 	-T!Z&,,	
{KL 	'K!c&,,		
MK` 	&J!b&,,		
aKt 	.V!n&,,		
uKH 	).!U&,,		
IK\ 	)K"x&,,	
]Kt 	*2!Y&,,	
uKB 	*2!Y#2,,	
CKT 	*2!Yb	
UKd 	##!DK ,,'(Z[	
eKv 	##!D\  ,,'(Z[		
wKJ 	&#!GK ,,	
KK\ 	&#!G\  ,,	
]Kn 	&;"yK ,,		
oKB 	&;"yK ,,		
CKV 	4D!VP ,,	
WKf 	3\"HP ,,	
gKx 	3\"HP ,,	
yKJ 	*V"f; !%	
KK\ 	S "C;' !%	
]Kl 	&!!D#,,	
mKz 	(!!F%,,	
{KH 	'BTXabv -,,	
IKZ 	'BTXabv 6	
[Kj 	'FQU\efv 3,,	
kK| 	'DrXabu 3,,	
}KN 	'BTX`puv9
 !,,,	
OKf 	'EPT[csxy9
 !2,,	
gK~ 	'DrX`puv9
 !2,,	
KV 	'BTX`puv9
 !5
	
WKl 	'BTX`ptu8
 !:,,	
mKD 	'DrX`ptu8
 !@,,	
EK\ 	'ADW`a: &<,,
	
]Kr 	'BTXab: $-,,
	
sKH 	'HSW^gh: $3,,
	
IK^ 	'DrXab: $3,,
	
_Kt 	'BTXab: $6		
uKH 	'BTXbrwx9
 $.,,	
IK` 	'BTXbrwx9
 $7
	
aKv 	'FQU-7uN9
 $4,,	
wKP 	'Dr-7uN9
 $4,,	
QKj 	'Dx-7uN9
 $A,,	
kKD 	'Dx-7uN9
 $A,,	
EK^ 	'FQU\fvz{8
 $B,,	
_Kv 	'Dx-7tM8
 $O,,	
wKP 	'BTXaqvw9
 $-,,	
QKh 	'BTXaqvw9
 $6
	
iK~ 	'FQU-6eM9
 $3,,	
KX  	'DrXaqvw9
 $3,,	
Y Kp  	'Dx-6eM9
 $@,,	
q KJ! 	'Dx-6eM9
 $@,,	
K!Kd! 	'FQU\euyz8
 $A,,	
e!K|! 	'Dx-6dL8
 $N,,	
}!KV" 	'BTXab: '-,,
	
W"Kl" 	'BTXab: '6		
m"K@# 	'LW[bkl: '3,,
	
A#KV# 	'DrXab: '3,,
	
W#Kl# 	'BTXcsxy9
 '/,,	
m#KD$ 	'BTXcsxy9
 '8
	
E$KZ$ 	'I59[`b9
 '5,,	
[$Kt$ 	'DrXcsxy9
 '5,,	
u$KP% 	'ITX-8N8
 'C,,	
Q%Kj% 	'DrXcswx8
 'C,,	
k%KF& 	'		r:8&,,	
G&KV& 	'		qNN,,,	
W&Kh& 	'		q97%.,,	
i&Kx& 	'		q97#&,,	
y&KH' 	'		qNN#,,,	
I'KZ' 	'		q97#.,,	
['Kl' 	'		q97#.,,	
m'K~' 	'		r:80	
'KN( 	'2>;*,,	
O(K^( 	'1=:%2,,	
_(Kn( 	'1=:#*,,	
o(K~( 	'1=:#',,	
(KN) 	'1=:#',,	
O)K^) 	'1=:)	
_)Kl) 	I!s# ,,	
m)K~) 	O!y! ,,	
)KP* 	.O!y ),,		
Q*Kd* 	'5!_# ,,	
e*Kv* 	%6!`! ,,	
w*KH+ 	26!`) ,,		
I+K\+ 	'!R&,,	
]+Kn+ 	*E,,	
o+Kx+ 	*E ,,		
y+KD, 	*D ,,		
E,KP, 	*/!gE(,,	
Q,K`, 	*=2,,	
a,Kn, 	1,"5 !"  % ,,	
o,KP- 	1AsAFF3"A ! "#.4D,,	
Q-Kl- 	1+"5 H ",,	
m-KF. 	1AsAFF3"A H ",,	
G.Kb. 	%Aq!S!&&9"A Z ##.4D,,!	
c.KF/ 	 #!A*Z ,,
	
G/K\/ 	!!X% 2,,	
]/Kn/ 	#!J %**aV"45 2,,	
o/K@0 	#*D%8!p& $-
	
A0K\0 +.9Y+[ #&&  	AAv,)3 E!QUO,G		#c7*; <<sBK$(*J 5(@(@#@@K##"&qc&'Aq'1a|%T-<QC @66A] C; <L:LA*Q  * +(>7"#'"',,!	#L'#:UU\\J:((3: 	%eV[VbVbc: 	ETYT`T`a	: 	uu||D: 	.: 	%,,/: 	.: 	5: 	_ell3: 	ou||4: 	5: 	%,,/:  	/5<<0!:" 	/5<<0#:$ 	%,,/%:& 	%,,/':( !%,,/#U\\2$ell3&5$ell3 %,,/&55:%8 "A 
6#-2 % 
 7::;PRTU
##J/ 45
6 r)   c                 6   |rt        j                  |      || z
  z  }n||j                         | z
  z  }|dk(  r|j                         S |dk(  r|j	                         S |dk(  r5|j                         dk7  r"|j	                         |j                  d      z  S |S )Nr  rS  	batchmeanr   )r5   r   r   r  rS  rQ  rd   )r  rK  r   r  r&   s        r'   kldivloss_referencer  
  s    6"fun56::<%/0F{{}	e	zz|	k	!fjjla&7zz|fkk!n,,Mr)   c                    | j                         dk\  sJ | j                  d      }| j                  d      }|f| j                         dd  z   }t        j                  |      j	                  |       }|$t        j
                  |      j	                  |       }d}	t        |D 
cg c]  }
t        |
       c}
 D ]X  }||   }||k(  rdn||   j                         }t        |      }|j                  d|       | t        |          |z  ||<   |	|z  }	Z |dk(  r|j                         |	z  S |dk(  r|j                         S |S c c}
w )NrW   r   r3   rM   r  r  rS  )rQ  rd   r5   r  r   onesr   r  itemr  insertr  rS  )r  rK  r+   rv  r   NCout_sizeoutputtotal_weightrd   tupt_nxnorminput_indexs                  r'   nlllossNd_referencer  
  sB   99;!

1A

1Atejjl12&&H[["**51F~A&&u-L:t:; c{!T)rvd|/@/@/B3i1d#U;/0047s Fzz|l**	e	zz|M ;s   $Ec                    | j                         dk\  sJ t        j                  | d      } | j                  d      }|$t        j                  |      j                  |       } |j                  d|gd | j                  dd  D         }|dkD  r|dk  sJ |d|z
  z  ||z  z   }| |z  |z  j                  d       }|dk(  r|j                         S |dk(  r|j                         S |S )	NrM   r3   c              3       K   | ]  }d   ywr  rl   r  r=   s     r'   r  z;cross_entropy_loss_prob_target_reference.<locals>.<genexpr>
  s      <q <   r  rX  rP  r  rS  )
rQ  r5   r  rd   r  r   r8   shaperS  r  )r  rK  r+   r   label_smoothingr  r  s          r'   (cross_entropy_loss_prob_target_referencer  
  s    99;!eQ'E

1A~A&&u-V[[A= <EKKO <=F#%%%A/0?Q3FFv~&+++22FF{{}	e	zz|Mr)   c                 (   t        j                  | d      }t        j                  |||||      }|dk(  r|S d|cxk  rdk  sJ  J t        j                  | d      } | j	                  d      }|+|  |j
                  d|gd | j                  dd  D         z  } t        j                  | d       }	||k(  }
|	j                  |
d       |dk(  r|bt        j                  |	      |j                  d|j                  |
j                               j                               j                         z  }nPt        j                  |	j                  |
j                                     }n|d	k(  rt        j                  |	      }n|	}d|z
  |z  |||z  z  z   S )
Nr3   ru  r  rX  c              3       K   | ]  }d   ywr  rl   r  s     r'   r  z>cross_entropy_loss_indices_target_reference.<locals>.<genexpr>
  s     +G!A+Gr  rM   r  r   rS  )r5   r  r   ra  rd   r8   r	  rS  masked_fill_gathermasked_selectlogical_notrZ   r  )r  rK  r+   rv  r   r
  log_softmax_inputnlllossr  smooth_lossignore_maskrets               r'   +cross_entropy_loss_indices_target_referencer  
  s   ))%3jj!G #'C'''''eQ'E

1AAqH+Gu{{12+GHH99UA&&KL(K[#.F ))K(6==F<P<PQ\QhQhQj<k<s<s<u+v+z+z+||C**[66{7N7N7PQRC	e	ii$7*SOa4G-HHHr)   c                 v    | j                   |j                   k(  rt        | ||||      S t        | |||||      S )N)r+   r   r
  )r+   r   rv  r
  )r	  r  r  )r  rK  r+   rv  r   r
  s         r'   cross_entropy_loss_referencer    sK    {{fll"7+- 	- ;6&I%
 	
r)   c           
          d }t        | |      D cg c]  \  }} |||||       }}}t        | \  }	}
| j                  |	      }|dk(  rt        |      t        |
      z  S |dk(  rt        |      S |S c c}}w )Nc                 :    ||k(  ry|dn||   }| |    |z  }||fS )Nr  r3   rl   )r  rK  r+   rv  r  r&   s         r'   nll_loss_helperz*nllloss_reference.<locals>.nll_loss_helper  s6    \!Nqv-$&~r)   r  rS  )r  
new_tensorrS  )r  rK  r+   rv  r   r  r;   r7   losses_and_weightslossesr,   losses_tensors               r'   nllloss_referencer!    s     '*%&8:"a *!QE : :-.OFG$$V,MF=!CL00	e	=!!:s   A5c                    | |z
  j                         }||k\  j                  |      }||k  j                  |      }|dk(  r|}n||d|z  z
  z  |dz  |dz  z  |z  z   }|dk(  r|j                         S |dk(  r|j                         S |S )Nr   r  rM   r  rS  )r3  r   r  rS  )r  rK  r   r  abs_diffge_beta_masklt_beta_maskr  s           r'   smoothl1loss_referencer&  +  s    ##%H$--h7LtO,,X6LqyC$J!67,:LPX\]P]:^ae:eeF{{}	e	zz|Mr)   c                     | |z
  j                         }||k\  }||k  }||z  |d|z  z
  z  |dz  |dz  z  z   }|dk(  r|j                         S |dk(  r|j                         S |S )Nr  rM   r  rS  )r3  r  rS  )r  rK  r   r  r#  ge_delta_masklt_delta_maskr  s           r'   huberloss_referencer*  ;  s    ##%H&M%MU"hu&<=PS@SW_cdWd@eeFF{{}	e	zz|Mr)   c           	          g }|D ]  }|dk  r n|j                  |        d}|D ]<  }t        dt        |             D ]"  }||vs|t        dd| |   z
  | |   z         z  }$ > |S )Nr   r3   )r  r  lenr  )r  rK  targetstarget_indexrS  r;   s         r'   _multilabelmarginloss_referencer/  G  s    G %!|$%
 C Bq#e*% 	BAs1a%"55a@AA	BB
 Jr)   c                    | j                         }| j                         dk  r|j                         dk  sJ | j                         dk(  r| j                  d      n| j                  d      j                  d      } |j                         dk(  r|j                  d      n|j                  d      j                  d      }| j                  d      }| j                  d      }| j                  |      j	                         }t        d|      D ]  }t        | |   ||         ||<    |dk(  r|j                         |z  S |dk(  r|j                         |z  S |dk  r|j                         |z  S ||z  S NrM   r3   r   r  rS  )
rQ  r  rd   newzero_r  r/  r  rS  r  )r  rK  r   	input_dimnrQ  r  r;   s           r'   multilabelmarginloss_referencer6  W  sO   		Iyy{Qzz|a&+iikQ&6"EOOA<N<X<XYZ<[(.

(9!!!$v?O?OPQ?R?\?\]^?_

1A
**Q-CYYq\!F1a[ I3E!HfQiHq	I F{{}s""	e	zz|c!!	Q ~~#%%|r)   c                     || z
  j                  d      j                  |       }t        j                  |dk(  | |      }|dk(  r|j	                         S |dk(  r|j                         S |S )Nr   r  r3   r  rS  )r"  r   r5   wherer  rS  )r  rK  r<  r   margin_clampr  s         r'   hingeembeddingloss_referencer:  q  sg    UN))a)088?L[[1e\:FF{{}	e	zz|Mr)   c                     d|  |z  j                         z   j                         }|dk(  r|j                         S |dk(  r|j                         S |S )Nr3   r  rS  )r   r   r  rS  )r  rK  r   r  s       r'   softmarginloss_referencer<  |  sQ    E6F?''))..0FF{{}	e	zz|Mr)   c           	          |)| j                  t        |             j                  d      }d}t        dt        |             D ],  }||k7  s	|||   t	        d|| |   z
  | |   z         |z  z  z  }. |S )Nr3   r   )r2  r,  fill_r  r  )r  
target_idxr<   r<  r+   r  r;   s          r'   _multimarginloss_referencer@    s    ~3u:&,,Q/F1c%j! b
?fZ(CFU:=N4NQVWXQY4Y,[_`,`aaFb Mr)   c                 l   | j                         dk  rD| j                         dk(  r| j                  d      n| j                  d      j                  d      } |j                         }|j                         dk(  r|j                  d      }| j                  d      }| j                  d      }| j                  |      }	t	        d|      D ]  }
t        | |
   ||
   |||      |	|
<    |dk(  r|	j                         |z  S |dk(  r|	j                         |z  S |dk(  r|	j                  d      |z  S |	|z  S r1  )	rQ  r  rd   r2  r  r@  r  rS  r  )r  rK  r<   r<  r+   r   
target_dimr5  rQ  r  xs              r'   multimarginloss_referencerD    s   yy{Q&+iikQ&6"EOOA<N<X<XYZ<[Jzz|q!!!$

1A
**Q-CYYq\F1a[ W.uQxAvvVq	W F{{}s""	e	zz|c!!	q~~a 3&&C<r)   c                     d }t        j                  |dk(  d || |      z
   || |      |z
  j                  d            }|dk(  r|j                         S |dk(  r|j	                         S |S )Nc                 8   | j                  | j                  d            }t        d| j                  d            D ][  }| |   ||   z  j                         | |   | |   z  j                         dz   ||   ||   z  j                         dz   z  dz  z  ||<   ] |S )Nr   g-q=r  )r2  rd   r  rS  )abcosr;   s       r'   _cosz+cosineembeddingloss_reference.<locals>._cos  s    eeAFF1Iq!&&)$ 	tAdQqTk&&(qtad{.?.?.AE.IqQRtVWXYVZ{N_N_NadiNi-jor,rsCF	t
r)   r3   r   r  r  rS  )r5   r8  r"  r  rS  )input1input2rK  r<  r   rJ  r  s          r'   cosineembeddingloss_referencerM    st     [[1a$vv*>&>ffAUX^A^@e@ejk@e@lmFF{{}	e	zz|Mr)   c                 T   t        j                  | |||      }t        j                  | |||      }	|r.t        j                  ||||      }
t        j                  |	|
      }	t        j                  ||z   |	z
  d      }|dk(  r|j	                         S |dk(  r|j                         S |S )Nr  r  r  rS  )r5   pairwise_distancer  r"  r  rS  )anchorpositivenegativer<  r<   epsswapr   d_pd_nd_sr  s               r'   tripletmarginloss_referencerX    s    

!
!&(As
;C

!
!&(As
;C%%h!SAiiS![[#+5FF{{}	e	zz|Mr)   c                     | | |z
  z  |z   j                  d      }|dk(  r|j                         S |dk(  r|j                         S |S )Nr   r  r  rS  )r"  r  rS  )rK  rL  rK  r<  r   r  s         r'   marginrankingloss_referencerZ    sR    g&)F299a9@FF{{}	e	zz|Mr)   c                 @   t        j                  |t         j                        }t        j                  |t         j                        }| j                  }| j	                         } |j                         }|j                  d      }g }t        | j                  d            D ]|  }	||	   j                         }
||	   j                         }||	   j                         }|j                  d|z  dz   f|      }|j                         dk(  r||	d |f   |dd d<   n|||z
  | |dd d<   | d |
|	f   j                         }| j                  |dz  dz   f      }|d|f   |d<   |d|d   f   |d<   |d d |dd  k7  }t        d|
      D ]b  }|j                         }|dd xxx |d d z  ccc |dd xxx t        j                  ||d d |j                  d            z  ccc |||f   |z  }d |j                  |dd  j!                         j#                         d             t        j$                  |d      }|dk(  r9||j'                  |j                  |j(                        z  j+                         }n|d	k(  r|j!                         }|j'                  |      }|S )
Nr   r   r3   rM   r  r4   r  r   devicerS  )r5   	as_tensorrb  r   rc   cumsumr  rd   r  new_fullrQ  r   	new_zeroscloner8  r  rS  r   catr   r]  r  )	log_probsr-  input_lengthstarget_lengthsblankr   dtcum_target_lengthsr  r;   input_lengthtarget_lengthcum_target_lengthtargets_primeprobsalpha
mask_thirdr7   
alpha_nextr  s                       r'   ctcloss_referencerr    s   OOMDM__^5::FN	B  "IllnG'..q1F9>>!$% 5$Q',,.&q)..0.q1668((!m*;a*?)A5I;;=A")!^m^*;"<M!$Q$")*;m*KL]"^M!$Q$-<-*+//1##]Q%6%:$<=E?aM!,,-a#CR(M!",==
q,' 	9AJqrNeCRj(NqrNekk*eCRj%//RSBTUUN!]*+j8E		9
 	uRSz~~'++-d334'5( YYvq!FF>,,6<<,VV\\^	e	YYr]FMr)   )r  )r   r  rj  r  r  r  r  r1  rF  rf  CosineEmbeddingLossTripletMarginLossMarginRankingLossCTCLossCrossEntropyLossr|  r   c                      | d   }d }fd | dd D cg c]
  } ||       c}      } || }t        |      }|dk(  r|j                  d      S |S c c}w )zReference function for criterion supporting no batch dimensions.

    The criterion is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    r4   c                     t        | t        t        f      r| D cg c]  }|j                  d       c}S | j                  d      S c c}w rJ   r  r  s     r'   r  z:single_batch_reference_criterion_fn.<locals>.unsqueeze_inp  r  r  c                     g }t        | t        t        f      r | D ]  }|j                   |              |S |j	                  |        |S rk   )r  r  r  extendr  )xsr&   rC  rZ   s      r'   rZ   z4single_batch_reference_criterion_fn.<locals>.flatten  sO    b4-( *gaj)*  MM"r)   Nr}   r   )r(   r  )rn   	criterionr  r  single_batch_input_argsr  r   rZ   s          @r'   #single_batch_reference_criterion_fnr    sr     RI 
 &cr&S}U';&ST/0Fi(IF~~a  M 'Ts   A)L1LossMSELossPoissonNLLLossr  r  )r}   r  rS  _no_batch_dim_)namec                 8     t        t        |       t              S Nr~   r!   rv   r   r  rn   s     r'   r>   r>   6      ->WR->-S r)   r  )r   r   rC   target_sizerD   r  rG   KLDivLoss_no_batch_dim_c                  6    t        j                  t              S r  )rv   r   r   rl   r)   r'   r>   r>   C  s    BLL9= r)   c                  H    t        j                  d      j                         S Nr  r   rl   r)   r'   r>   r>   D  s    D)--/ r)   c                  ,    t        j                  d      S r  r   rl   r)   r'   r>   r>   E  s    %**T* r)   )r   r   r   	target_fnrD   r  rG   BCELossc                  r    t        j                  t        j                  dt         j                              S Nr8  r   )r5   r   r   rc   rl   r)   r'   r>   r>   R  s    ekk!5<<@A r)   c                      t        j                  dt         j                        j                  d      j	                  t         j                        S )Nr8  r   r   )r5   r   rc   r   r   rl   r)   r'   r>   r>   S  s/    AU\\255a8;;ELLI r)   BCEWithLogitsLossc                  L    t        j                  dt         j                        S r  r5   r   rc   rl   r)   r'   r>   r>   U  s    %++au||"D r)   c                  L    t        j                  dt         j                        S r  r  rl   r)   r'   r>   r>   U  s    ekkZ[chcocoNp r)   r1  c                  L    t        j                  dt         j                        S r  r  rl   r)   r'   r>   r>   V  s    5;;q#E r)   c                  6    t        j                  g ddz        S N)r4   r3   r3   rW   r5   tensorrl   r)   r'   r>   r>   V  s    u||\fij\jOk r)   r  c                  L    t        j                  dt         j                        S )Nr1   r   r  rl   r)   r'   r>   r>   W  s    U[[%,,%G r)   c                  0    t        j                  g d      S )N)rW   r   r4   r3   r  rl   r)   r'   r>   r>   W  s    QVQ]Q]^kQl r)   rF  c                  L    t        j                  dt         j                        S r  r  rl   r)   r'   r>   r>   X  s    u{{1ELLA r)   c                  6    t        j                  g ddz        S r  r  rl   r)   r'   r>   r>   X  s    5<<XbefXfKg r)   rj  c                  v    t        j                  t        j                  dt        j                        d      S )NrW   r   r   rP  )r   r  r5   r   rc   rl   r)   r'   r>   r>   Y  s     ekk!5<<&HaP r)   c                  ,    t        j                  d      S rY   r  rl   r)   r'   r>   r>   Y  s    Z_ZfZfghZi r)   rs  c                      t        j                  dt         j                        t        j                  dt         j                        fS r  r  rl   r)   r'   r>   r>   \  )    Qell3U[[%,,5WX r)   c                  L    t        j                  dt         j                        S )Nr3   r   )r5   r  rc   rl   r)   r'   r>   r>   ]  s    Qell3 r)   ru  c                  V    t        j                  d      t        j                  d      fS r  r/  rl   r)   r'   r>   r>   `  s    5;;r?EKKO"D r)   c                  H    t        j                  d      j                         S r  )r5   r   signrl   r)   r'   r>   r>   `  s    ekkZ\oNbNbNd r)   rt  c                      t        j                  dt         j                        t        j                  dt         j                        fS r  r  rl   r)   r'   r>   r>   d  r  r)   c                  L    t        j                  dt         j                        S r  r  rl   r)   r'   r>   r>   e  s    AU\\2 r)   MultiLabelSoftMarginLossc                  L    t        j                  dt         j                        S r  r  rl   r)   r'   r>   r>   g  s    Qell)K r)   c                  ,    t        j                  d      S )Nr8  r/  rl   r)   r'   r>   r>   g  s    UZU`U`abUc r)   r_   ,classification_criterion_no_batch_extra_info)r  r  r1  rj  rF  c                 8     t        t        |       t              S r  r  r  s     r'   r>   r>   y  r  r)   c                      |        S rk   rl   fs    r'   r>   r>   z  s    AC r)   c                      |        S rk   rl   r  s    r'   r>   r>   {  s    ac r)   )r   r   r   r  rD   r  
has_parityc                   |   e Zd Zed        Zedej                  deeej                     eej                     f   fd       Z
edej                  ddfd       Ze	 ddej                  dedej                  d	eej                  eej                     f   d
ef
d       Zd Zd Zd ZddefdZddefdZddefdZy)
NNTestCasec                     t         rk   NotImplementedError)rm   rn   rp   s      r'   _forwardzNNTestCase._forward      !!r)   r  returnc                     t         rk   r  rm   r  s     r'   _get_parameterszNNTestCase._get_parameters  r  r)   Nc                     t         rk   r  r  s     r'   _zero_grad_parametersz NNTestCase._zero_grad_parameters  r  r)   r  r  grad_outputcreate_graphc                     t         rk   r  )rm   r  r  r  r  r  s         r'   	_backwardzNNTestCase._backward  s
    
 "!r)   c                     t        |t              rt         fd|D              S t        |t              r |D cg c]  } j                  |       c}S t	        j
                  |j                               S c c}w )Nc              3   B   K   | ]  }j                  |        y wrk   )	_jacobian)r  elemnum_outrm   s     r'   r  z'NNTestCase._jacobian.<locals>.<genexpr>  s     I4g6Is   )r  r  r  r  r5   r  nelement)rm   r  r  r  s   ` ` r'   r  zNNTestCase._jacobian  sb    eU#I5IIIt$>CDdDNN41DD;;u~~/99 Es   A<c                      t        |t        j                        r<|j                  r|j	                         j                  d      S |j                  d      S t         fd|D              S )Nr4   c              3   @   K   | ]  }j                  |        y wrk   )_flatten_tensors)r  rG  rm   s     r'   r  z.NNTestCase._flatten_tensors.<locals>.<genexpr>  s     =a..q1=   )r  r5   r  	is_sparseto_denser8   r  )rm   rC  s   ` r'   r  zNNTestCase._flatten_tensors  sN    a&{{zz|((,,vvbz!=1===r)   c                    t        |t        j                        rO|j                  rB|j                  5|j                  j                          |j                  j                          y y y |D ]  }| j                  |        y rk   )r  r5   r  r  gradr3  detach__zero_grad_input)rm   r  r;   s      r'   r  zNNTestCase._zero_grad_input  si    eU\\*""uzz'=

  "

""$ (>"  )%%a()r)   c                    | j                  ||      }|j                         }|r&| j                  ||      }t        t	        |            }|r:t        d | j                  |      d   D              }	t        j                  |	|      }
t        |      D ]%  }| j                  |      \  }}t        ||      D cg c]  \  }}|t        j                  |      n|  }}}t        j                  |      }|j                  d      }d||<   |r| j                  |       |r| j                  |       | j                  ||||      }|rCt        t	        |            D ]+  \  }}|j!                         j                  d      |d d |f<   - |st        j"                  | j%                  |      d      
d d |f<   ( d}|r|fz  }|r|
fz  }|S c c}}w )Nc              3   <   K   | ]  }|j                           y wrk   )numelr  s     r'   r  z2NNTestCase._analytical_jacobian.<locals>.<genexpr>  s     O!AGGIO   r   r4   r3   rl   )r  r  r  r  r   rS  r  r5   r  r  r  
zeros_liker8   r  r  r  
contiguousrc  r  )rm   r  r  jacobian_inputjacobian_parametersr  r  jacobian_inpflat_jacobian_input	num_paramjacobian_paramr;   paramd_paramr<   r  d_out
flat_d_outd_input
jacobian_xd_xress                         r'   _analytical_jacobianzNNTestCase._analytical_jacobian  s   vu-oo'>>%=L"&}\'B"COt/C/CF/KA/NOOI"[[K@N{# 	TA!11&9NE7MPQVX_M`a6Aqaiu''*Q>aGa$$V,EBJJqM"**62%%e,nnVUFEBG'*+>g@V'W AOJ'*~~'7'<'<R'@Jq!t$A"',yy1F1Fw1OQR'Sq!t$)	T, )+<= C>?"C
3 bs   9#Gc                       fd}d}|r|t        ||d      fz  }|r] j                        \  }}g }	|D ](  }
t        |||
d      }|	j                  |d   d          * |t        j                  |	d      fz  }|S )Nc                  D    j                  |       j                         S rk   )r  detach)r  r  rm   s    r'   fwz*NNTestCase._numerical_jacobian.<locals>.fw  s    ==/6688r)   rl   ư>)rS  )rK  rS  r   )r   r  r  r5   rc  )rm   r  r  r  r  r  r  r  r=   to_catr<   jacobians   ``          r'   _numerical_jacobianzNNTestCase._numerical_jacobian  s    	9 )+*2u$?@@C++F3HE1F .22uQDQhqk!n-. EIIfa(**C
r)   c                 
   t        | j                  |      d         }| j                  ||||      }| j                  ||||      }t	        t        |            }t	        t        |            }g }	t        ||      D ]W  \  }
}|
j                         dk7  s|	j                  |
j                  |d      j                         j                                Y t        |	      dkD  r | j                  t        |	      t               y y )Nr   r4   )ro  )boolr  r  r  r  r   r  r  r  r  r3  r  r,  assertLessEqual	PRECISION)rm   r  r  r  r  
analytical	numericalanalytical_tnumerical_tdifferencesrG  r5  s               r'   check_jacobianzNNTestCase.check_jacobian  s    "4#7#7#?#BC..vunNab
,,VUNL_`	M*56=34k2 	CDAqwwyA~""155"5#5#9#9#;#?#?#AB	C {a  [!19=  r)   F)TTT)rr   rs   rt   r   r  rv   rw   r  r  	Parameterr  r  r   r5   r  r   r   r  r  r  r  r  r  r  r  rl   r)   r'   r  r    s    " " "bii "E$r||:LdSUS_S_N`:`4a " " "BII "$ " "  (-"		 ")"38<<"$U\\8ELL3I%IJ" !%" ":>)(2B (T1A ">,< >r)   r  c                   \    e Zd Zh dZddZd Zd Zed        Zed        Z	d Z
dd	Zd
 Zy)TestBase>   r  
extra_argsrA   Nc                     || _         || _        || _        || _        | j                  D ]?  }||vs|dz   |vs|dz   |vs|dv rd||<   "t        | j                          d| d       || _        i | _        y )N_fn_size>   r  rA   rl   z
: Specify z5 by a value, a function to generate it, or it's size!)	rK   r   r   rD   _required_arg_names
ValueErrorget_name_extra_kwargs
_arg_cache)rm   r   rK   rD   r   rp   r  s          r'   __init__zTestBase.__init__  s    	 &(,, 	@D6!dUl&&@TG^[aEa==#%F4L$'8
4&H}%~	@ $r)   c                     | j                   d| j                   z   S d| j                  j                  z   }| j                  r|d| j                  z   z  }|S )Ntest_r=   )r   r   rr   rK   )rm   	test_names     r'   r	  zTestBase.get_name  sP    ==$T]]**d..777	99tyy(Ir)   c                      t        |t        j                        r|S t        |      r t	        |       fd|D              S |S )Nc              3   @   K   | ]  }j                  |        y wrk   )_unpack)r  vrm   s     r'   r  z#TestBase._unpack.<locals>.<genexpr>  s     >1t||A>r  )r  r5   r  r   type)rm   values   ` r'   r  zTestBase._unpack  s:    eU\\*L4;>>>>Lr)   c                 &    | j                  dd      S )NrA   T_get_argrm   s    r'   rA   zTestBase.constructor_args#  s    }}/66r)   c                 &    | j                  dd      S )Nr  Tr  r  s    r'   r  zTestBase.extra_args'  s    }}\400r)   c           
         || j                   v sJ || j                  vr|dz   }|dz   }|| j                  v r| j                  |   | j                  |<   n|| j                  v r" | j                  |          | j                  |<   nV|| j                  v s!J d| d| d| d| j                                 fd | j                  |         | j                  |<   |r| j	                  | j                  |         S | j                  |   S )Nr  r  z	Missing `z`, `z` or `z` for c                     t        | t              r| D cg c]
  } |       c}S t        | t        j                        r| j	                         S t        j
                  |       S c c}w rk   )r  r  r5   r  rc   r   )sizessmap_tensor_sizess     r'   r  z+TestBase._get_arg.<locals>.map_tensor_sizes:  sS    !%.=BC 0 3CC#E5<<8$||~-${{511	  Ds   A')r  r  r
  r	  r  )rm   r  unpackfn_name	size_namer  s        @r'   r  zTestBase._get_arg+  s   t/////t&UlGwIt)))(,(:(:4(@%D...(C(:(:7(C(E% D$6$66 \vT)F7)6$--/IZ[\62 )99K9KI9V(W%6<t||DOOD12W$//RVBWWr)   c                 &    | j                  d|      S )Nr  r  )rm   r   s     r'   
_get_inputzTestBase._get_inputF  s    }}Wf--r)   c                     t         rk   r  )rm   	test_cases     r'   __call__zTestBase.__call__I  s    !!r)   ) NNr  )rr   rs   rt   r  r  r	  r  propertyrA   r  r  r$  r'  rl   r)   r'   r  r     sO    E 7 7 1 1X6."r)   r  c            	       l     e Zd Zededej                  dedefd       Z fdZd Z	d Z
d	 Zd
 Z xZS )
ModuleTestr&  r  r  r  c                     t         rk   r  )rm   r&  r  r  s       r'   _do_testzModuleTest._do_testO  r  r)   c                    t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd	      | _        |j                  d
d      | _	        |j                  dd       | _
        | j                  t        j                         | _
        y y )Nr  TrN   r   r_   !FIXME_no_cuda_gradgrad_comparisonFr   g-C6*?check_forward_onlyrG   )superr  r  r  should_test_cudashould_test_pickler_   r/  r   r0  rG   r5   get_default_dtyperm   rn   rp   	__class__s      r'   r  zModuleTest.__init__S  s    $)&)$jj)94@ &

; ="(**Xt"<$jj)94@JJ:EB 	.K6"(**-A5"I#ZZ>%!&!8!8!:D &r)   c           	      j   t        | j                        5   | j                  | j                   }| j	                         }| j
                  a|j                  ||      }t        |      }t        |      }| j                  ||j                  |      d   |      }|j                  ||d       | j                  r
	 d d d        y | j                  |||       | j                  rt        j                         5 }|j                  ||       t        j                   ||       |j#                  d       t        j$                  |d      }	|j                  |j                  ||      |j                  |	|             d d d        | j'                  |||       d d d        y # 1 sw Y   %xY w# 1 sw Y   y xY w)Nr   F)exact_dtype)weights_only)r   rG   r   rA   r$  rD   r  r   r  assertEqualr0  test_noncontigr3  tempfileTemporaryFiler5   saveseekloadr-  )
rm   r&  r  r  out	ref_input
ref_moduleexpected_outr  module_copys
             r'   r'  zModuleTest.__call__a  s   t112 	4%T%%t'<'<=FOO%E  ,((7$UO	%f-
#00I<U<UV\<]^_<`blm%%c<U%K&&	4 	4 	659&&++- u&&vu5JJvq)FF1I"'**QU"CK)))*<*<VU*KYM_M_`kmrMstu MM)VU31	4 	4 u u!	4 	4s+   B$F)3F)6BF9F)F&	"F))F2c                 Z    t        |t              r|D cg c]  } j                  |       c}S t        |t              rt         fd|D              S |}|j	                         }|}t        |      D ]  }|j                  |      dkD  s|dz   } n t        j                  t        j                  |      |g|      j                  |d      j                         }|j                         dk(  s%|j                         dk(  s|j                         rJ |j                  |_        |S c c}w )Nc              3   @   K   | ]  }j                  |        y wrk   )noncontiguize)r  orm   s     r'   r  z+ModuleTest.noncontiguize.<locals>.<genexpr>  s     <1++A.<r  r3   r   )r  r  rH  r  rQ  r  rd   r5   r"  
empty_likeselectr  r  is_contiguousr  )rm   objrI  r  ndimrQ  r  	noncontigs   `       r'   rH  zModuleTest.noncontiguize|  s   c4 367aD&&q)77U#<<<<zz|
 t 	A{{1~!!e	 KK!1!1&!96 BCHOOPSUVW^^`	 A%):a)?yG^G^G```"("6"6	# 8s   D(c           	      X   t        |t        j                        r|j                         dk(  ry t	        d |D              ry |j                  |       |j                  |       t               5  |j                  ||      }t        |dd      r|d   }|j                  |j                        j                         }|j                         }t        |j                  ||||            }t        |j!                  |      d         }d d d        | j#                  |      }| j#                        }	t%        dd      D ]  \  }
}|
r|n|}t        |r|n|	      }|j                  |       |j                  |       t               5  |j                  ||      }t        |dd      r|d   }|j                  ||||      }|j'                  |       |j'                  |d	d
       |j'                  |j!                  |      d          d d d         y # 1 sw Y   xY w# 1 sw Y   xY w)Nr   c              3   x   K   | ]2  }t        |t        j                        s|j                         d k(   4 yw)r   N)r  r5   r  rQ  )r  r;   s     r'   r  z,ModuleTest.test_noncontig.<locals>.<genexpr>  s'     JjELL.Iquuw!|Js   ::return_indicesFr3   TFrM   )r1  g-C6?atolrtol)r  r5   r  rQ  anyr  r  r   r  r!   r2  r	  normal_rb  r   r  r  rH  r   r:  )rm   r&  r  r  r  r  r  r  nc_inputnc_grad_outputcontig_icontig_gr;   gorA  r  s                   r'   r;  zModuleTest.test_noncontig  s   eU\\*uyy{a/?JUJJ''/""5) 	E''6Fv/7 **V\\2::<K\\^Fy2265&+VWGy88@CDG	E %%e,++K8")-"B 	UHh!xA +~FB++F3&&q)!# U((36#3U;a&C **61c2>%%c62%%dG$Q%G%%i&?&?&G&JGTU U	U	E 	E$U Us   .BHBH H H)	c           	      
   t         r| j                  st        j                  d      t	        | j
                        5  | j                         }t        j                  t        j                  i}t        |t              r|n|f}t        d |D              }t        ||      } | j                  | j                   } | j                  | j                   j                         j!                         }|j#                  |      }	|j#                  |      }
t%        |	d   |
d         D ]   \  }}|j&                  j)                  |       " |j+                  |       |j+                  |       |j-                  |       |j-                  |       |j/                  ||      }|j/                  ||      }t1        |dd      r
|d   }|d   }|j3                  ||| j4                  dd       t7        d      D ]  }|j9                         j;                         }|j=                  |      }|j?                  ||||      }|j?                  ||||      }|j3                  ||| j4                  dd       t%        |	d	   |
d	         D ]$  \  }}|j3                  ||| j4                  d
       &  | j@                  r| jB                  s || } || }t1        |dd      r
|d   }|d   }t        jD                  |d      }|j=                  |      jG                         }d|_$        t        jJ                  jM                  ||t        |jO                               z   |d      }t        jJ                  jM                  ||t        |jO                               z   |d      }t%        ||      D ]%  \  }}|j3                  ||| j4                  dd       ' |rc|jQ                         jS                         tQ        d |D              z   }|jQ                         jS                         tQ        d |D              z   }nF|jQ                         tQ        d |D              z   }|jQ                         tQ        d |D              z   }t        jJ                  jM                  |||fz   t        |jO                               z   d      }t        jJ                  jM                  |||fz   t        |jO                               z   d      }|j3                  | j4                  dd       t%        ||      D ]%  \  }}|j3                  ||| j4                  dd       ' | jU                  |||       d d d        y # 1 sw Y   y xY w)NExcluded from CUDA testsc              3      K   | ]6  }t        |t        j                        xr |j                  j                   8 y wrk   )r  r5   r  r   
is_complexr  r7   s     r'   r  z'ModuleTest.test_cuda.<locals>.<genexpr>  s.     &s^_z!U\\'B'YqwwGYGY'Y&ss   <>)type_mapr   rR  FrU  rV  r8  rR   r3   rT  T)r  )r  c              3   X   K   | ]"  }|j                         j                          $ y wrk   rS  r3  r  rC  s     r'   r  z'ModuleTest.test_cuda.<locals>.<genexpr>       >eQRquuw{{}>e   (*c              3   X   K   | ]"  }|j                         j                          $ y wrk   rf  rg  s     r'   r  z'ModuleTest.test_cuda.<locals>.<genexpr>   rh  ri  c              3   <   K   | ]  }|j                           y wrk   rS  rg  s     r'   r  z'ModuleTest.test_cuda.<locals>.<genexpr>       8YQ8Yr  c              3   <   K   | ]  }|j                           y wrk   rl  rg  s     r'   r  z'ModuleTest.test_cuda.<locals>.<genexpr>  rm  r  )retain_graph)+r   r2  unittestSkipTestr   rG   r$  r5   rc   floatr  r  rW  r   r   rA   cudar  r  r  copy_r  r  r  r!   r:  r   r  rb  rX  r   r  r_   r/  
randn_liker  r  autogradr  r  rS  r3  r;  )rm   r&  	cpu_inputrc  cpu_input_tupleis_any_input_complexgpu_input_tuple
cpu_module
gpu_module	cpu_param	gpu_paramcpu_pgpu_p
cpu_output
gpu_outputr=   cpu_gradOutputgpu_gradOutputcpu_gradInputgpu_gradInputcpu_d_pgpu_d_pcpu_gradInputsgpu_gradInputscpu_d_igpu_d_ioutputs_cpuoutputs_gpucpu_gggpu_ggs                                 r'   rN   zModuleTest.test_cuda  s8    5 5##$>??t112 W	H)Iekk2H+5i+Gii\O#&&scr&s#s $_xHO)))4+@+@AJ)))4+@+@AGGINNPJ!11*=I!11*=I #IaL)A, ? (u

  '( &&7&&7++J7++J7"++JHJ"++JHJz#3U;']
']
!!*jt~~TUch!i 1X Y!+!1!1!3!;!;!=!/!7!7
!C ) 3 3JQ[]k l ) 3 3JQ[]k l%%m]^_mr%s(+IaL)A,(G Y$GW))'7VW)XYY ""4+Q+Q'9
'9
:'7?!+AJ!+AJ!&!1!1*D!Q!/!7!7
!C!J!J!L/3,!&!4!4#eJ,A,A,C&DD"!%	 "5 "'
 "'!4!4#eJ,A,A,C&DD"!%	 "5 "' ),NN(K l$GW))'7VWej)kl (",.."2"6"6"83>eVd>e;e"eK",.."2"6"6"83>eVd>e;e"eK",.."2S8Y.8Y5Y"YK",.."2S8Y.8Y5Y"YK,,#~&77%
@U@U@W:XX!% - ' ,,#~&77%
@U@U@W:XX!% - ' %%m]^_mr%s(+FF(; l$GW))'7VWej)kl 	:GoW	H W	H W	Hs   T!U''U0)rr   rs   rt   r   r   rv   rw   r-  r  r'  rH  r;  rN   __classcell__r6  s   @r'   r+  r+  M  sR    "# "ryy " " " ";46*#UJ[Hr)   r+  c                       e Zd Zd Zy)InputVariableMixinc                 J    t         j                  | d      }fd |      S )NFc                     t        | t        j                        r)| j                         s| j	                         rd| _        | S  t        |       fd| D              S )NTc              3   .   K   | ]  } |        y wrk   rl   )r  r  map_variabless     r'   r  zGInputVariableMixin._get_input.<locals>.map_variables.<locals>.<genexpr>  s     At}T2A   )r  r5   r  is_floating_pointra  r  r  )r;   r  s    r'   r  z4InputVariableMixin._get_input.<locals>.map_variables  sH    !U\\*&&(ALLN&*AOtAwAqAAAr)   )r  r$  )rm   r  r  s     @r'   r$  zInputVariableMixin._get_input  s(    ##D%0	B U##r)   N)rr   rs   rt   r$  rl   r)   r'   r  r    s    $r)   r  c                   @     e Zd Z fdZd Zd Zd Zed        Z xZ	S )NewModuleTestc                 z   t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  d	d
      | _	        |j                  dd      | _
        |j                  dd      | _        |j                  dd      | _        |j                  dd       | _        |j                  dd      | _        |j                  dd      | _        y )Nr  Fcheck_inplacer_   Tskip_doubler  rE   rF   r\   test_cpur3  r`   r  r   supports_fwgrad_bwgrad)r1  r  r  r  r  r_   r  r  rE   rF   r  r3  r`   r  r   r  r5  s      r'   r  zNewModuleTest.__init__$  s   $)&)ZZ/
#ZZ?$jj)94@!::mU;K7K7$jj)95A

:t4$*JJ/Eu$M!"(**-A4"H#)::.CT#J #)::.CU#K &,jj1I5&Q#r)   c           	         t        d j                         D              }t        |      fd}| j                  r6dk(  sJ t	        j
                  |d         }j                  |d   |       n@j                  t        |||z   | j                  | j                  | j                               | j                  rAj                  t        |||z   | j                  | j                  | j                               y y )Nc              3       K   | ]  }|  y wrk   rl   rg  s     r'   r  z1NewModuleTest._check_gradients.<locals>.<genexpr>5  s     6Qq6r  c                  6    |rJ j                  | d        S rk   )r  )inputs_and_paramsrp   r  
num_inputsr&  s     r'   fn_to_gradcheckz7NewModuleTest._check_gradients.<locals>.fn_to_gradcheck8  s&    :%%f.?.LMMr)   r3   r   )r`   	fast_modecheck_forward_ad)r`   r  check_fwd_over_rev)r  r  r,  r3  r5   r  r  
assertTruer   r`   r  r   r_   r   r  )rm   r&  r  input_tupleparamsr  test_input_jacobianr  s    ``    @r'   _check_gradientszNewModuleTest._check_gradients4  s    6&"3"3"566%
	N $$?"?"'"9"9+a."I$$V[^=PQ  ?K&<P>B>U>U595M5M<@<T<T"V W
   f@TBFBYBY9=9Q9QBFB]B]"_ ` r)   c                    t        j                         }t        j                  d       t        |t              r|n|f}| j                  |       j                          | j                  rt        |      dk(  sJ |d   } | j                  | j                  ddi}|j                  }t               5   |      }d d d        j                  |j                  |       t        |      }	|	j                         }
t               5   ||
      }d d d        j!                  |
j                  |       j                         |j"                  j                         j%                         }|j&                  7t        j(                         5  |j&                  j+                          d d d        |	j&                  7t        j(                         5  |	j&                  j+                          d d d        |j-                  |       |j-                  |       j                  |j&                  |	j&                         dfd	}t/        d |D              rt0        rt	        d |D              }j3                         j5                           |   |t         j4                  j6                  d       t         j4                  j9                         dkD  rt	        d |D              }j5                  d       t         j4                  j;                  d      5   |  d d d         |t         j4                  j6                  d       n@d	 fd
fdfdt	        fd|D              }j3                           |   |t         j6                         t	        fd|D              }j=                           |   |t         j>                         t0        r| j@                  rt	        fd|D              }j3                         j5                           |   |t         j4                  j6                  d       t	        d |D              }jC                           |   |t         j6                         t	        d |D              }j5                           |   |t         j4                  j6                  d       | jD                  rYt         jF                  jD                  jI                  d      5   |   |t         j4                  j6                  d       d d d        t         j4                  j9                         dk\  rqt	        d |D              }j5                  d       t         j4                  j;                  d      5   |  d d d         |t         j4                  j6                  d       | jJ                  sXt	        fd|D              }j=                         j5                           |   |t         j4                  j>                  d       | jL                  sXt	        fd|D              }jO                         j5                           |   |t         j4                  jP                  d       t        j                  |       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   WxY w)Nr3   r   inplaceTc                     j                         D ]7  }j                  ||        |j                  |j                         |       9 y rk   )r  assertIsInstancer:  
get_device)tensor_type	device_idr<   r  r&  s      r'   assert_module_parameters_arez<NewModuleTest._do_test.<locals>.assert_module_parameters_are{  sI    &&( E**1k:())!,,.)DEr)   c              3   P   K   | ]  }t        |t        j                           y wrk   )r  r5   
LongTensorrb  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     D1z!U--.Ds   $&c              3   <   K   | ]  }|j                           y wrk   rs  rb  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     >Q>r  c              3   >   K   | ]  }|j                  d         ywr  r  rb  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #C!AFF1I#C   c                     | j                         r| j                  |      S | j                         r| j                  |      S | S rk   )ra  r   r  )r  realr  s      r'   to_typez'NewModuleTest._do_test.<locals>.to_type  s<    $$&!99W----/!99T?*!Mr)   c                 4     | t         j                  d       S rk   )r5   float16rC  r  s    r'   to_halfz'NewModuleTest._do_test.<locals>.to_half  s    q%--66r)   c                 P     | t         j                  t         j                        S rk   )r5   float32	complex64r  s    r'   	to_singlez)NewModuleTest._do_test.<locals>.to_single  s    q%--AAr)   c                 P     | t         j                  t         j                        S rk   )r5   r  r  r  s    r'   	to_doublez)NewModuleTest._do_test.<locals>.to_double  s    q%--1A1ABBr)   c              3   .   K   | ]  } |        y wrk   rl   r  r7   r  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>       B	!Br  c              3   .   K   | ]  } |        y wrk   rl   r  r7   r  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  r  r  c              3   J   K   | ]  } |      j                           y wrk   r  r  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #MAIaL$5$5$7#M    #c              3   <   K   | ]  }|j                           y wrk   )cpurb  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #AAEEG#Ar  c              3   <   K   | ]  }|j                           y wrk   r  rb  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #BAFFH#Br  F)enabledrM   c              3   >   K   | ]  }|j                  d         ywr  r  rb  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Gaq	'Gr  c              3   J   K   | ]  } |      j                           y wrk   r  r  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Q	!(9(9(;'Qr  c              3   J   K   | ]  } |      j                           y wrk   r  )r  r7   r  s     r'   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Oa
(9'Or  rk   ))r5   get_num_threadsset_num_threadsr  r  r  __repr__r  r,  r   rA   _versionr   r:  r   rb  assertNotEqualr  rX  r  no_gradr3  backwardallr   rr  rs  FloatTensordevice_countr]  rc   DoubleTensorr2  r  r  backendsflagsr  r  half
HalfTensor)rm   r&  r  r  num_threadsr  	module_ipinput_versionr  input_ipinput_ip_clone	output_ipr  r  r  r  r  r  s    ``           @@@@r'   r-  zNewModuleTest._do_testP  sU   ++-a )%7eeXi= 	 {#q(((NE((($*?*?NNI!NNM!# ''!!%..-@H%^^-N!# 6%n5	6$$^%<%<mL!!&)4;;$$&..0Dzz%]]_ 'JJ$$&'}}(]]_ *MM'')*OOD!t$!!%**hmm<	E DDD  >+>>KLLN!K ()?)?Czz&&(1,##C{#CCAZZ&&q) )K(),UZZ-C-CQG"7BC  BkBBKLLNK ():):;  BkBBKMMOK (););<T22
 $#M#MM##%$,UZZ-C-CQG $#A[#AA

$,U->->? $#Bk#BB$,UZZ-C-CQG ::--33E3B P,4UZZ5K5KQOP ::**,1 #('G;'G"GKKKN**1- -,-01G1GK''"''Q['Q"QKMMO((*K(01H1H!L ~~"''O;'O"OKKKM&&(K(01F1FJk*w' '6 6' '* *.) )vP P- -sT   <	Z#	Z0Z=[
['[$[1#Z-0Z:=[
[[!$[.1[;c                 &    | j                  dd      S NrK  Fr  r  s    r'   _get_targetzNewModuleTest._get_target      }}Xu--r)   c                 &    | j                  dd      S NrA   Fr  r  s    r'   rA   zNewModuleTest.constructor_args      }}/77r)   )
rr   rs   rt   r  r  r-  r  r)  rA   r  r  s   @r'   r  r  #  s/    R `8Q+f. 8 8r)   r  c                        e Zd Zej                  j                  dh      Z fdZd ZddZd Z	e
d        Ze
d        Z xZS )	CriterionTestrK  c                 j   t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  d	d      | _	        |j                  d
d      | _
        |j                  dd      | _        |j                  dd      | _        |j                  dd       | _        | j                  t        j                         | _        y y )NrN   Tr0  Fr_   r  check_bfloat16check_complexr  rE   rF   r\   r`   rG   )r1  r  r  r2  r0  r_   r  r  r  r  rE   rF   r`   rG   r5   r4  r5  s      r'   r  zCriterionTest.__init__  s
   $)&) &

; ="(**-A5"I$jj)94@ **\48$jj)95A#ZZ?

:t4K6$jj)95A"(**-A4"H#ZZ>%!&!8!8!:D &r)   c                 (  
 t        | j                        5   | j                  | j                   
| j	                         }
j                          t        
       | j                         }| j                  g|j                  
||| j                        }t        |      t        |      f| j                  z   
fz   } | j                  | }|j                  ||       | j                  r
	 d d d        y t        d 
j                         D              }t!        |t              s|f|z   |fz   }
fd}	n||z   |fz   }
fd}	t#        |	|| j$                         | j&                  rt)        |	|| j$                         d d d        y # 1 sw Y   y xY w)Nr  c              3       K   | ]  }|  y wrk   rl   rg  s     r'   r  z)CriterionTest.__call__.<locals>.<genexpr>  s     :1:r  c                      | |      S rk   rl   )r  rK  r  r  s      r'   apply_fnz(CriterionTest.__call__.<locals>.apply_fn  s    !%00r)   c                      | ||      S rk   rl   )rK  rL  rK  r  r  s       r'   r  z(CriterionTest.__call__.<locals>.apply_fn  s    !&&&99r)   )r`   )r   rG   r   rA   r$  r  r|  r  rD   _forward_criterionr  r   r:  r0  r  r  r  r   r`   r_   r   )rm   r&  r  rK  rA  ref_argsrD  r  inputsr  r  s             @r'   r'  zCriterionTest.__call__   s~   t112 "	\%T%%t'<'<=FOO%E OOK%%'F  ,2265&UYUdUd2e$UOXf-=>PTZS\\0t00(;%%c<8&&#"	\ "	\& :f&7&7&9::FeU+F*fY61 6)3: h4;R;RS""h4CZCZ[E"	\ "	\ "	\s   CF5B
FFc                    d
fd	t         r| j                  st        j                  d      t	        | j
                        5  | j                         }| j                         } | j                  | j                   } | j                  | j                   } ||d      }|j                         s|j                         r	 ||      }|j                  |       |j                  |       t        |      }t        |      }	|j                          |t        j                   t        j"                  hv r9| j                         }| j                         } | j                  | j                   }|j%                  ||||      }
|j%                  |||	|      }|j'                  |
||t        j                   t        j"                  hv rdnddd	       |j)                  |||
||      }|j)                  ||||	|      }|j'                  |||t        j                   t        j"                  hv rdnddd	       d d d        y # 1 sw Y   y xY w)NFc                     t        | t        j                        r/| j                         j	                        j                        S t        | t              rt        fd| D              S | S )Nr   c              3   2   K   | ]  } |        y wrk   rl   )r  rI  convert_dtyper   r  s     r'   r  zACriterionTest.test_cuda.<locals>.convert_dtype.<locals>.<genexpr>*  s     Q]1e]CQs   )r  r5   r  r  r   requires_grad_r  )rM  r   r  r  s    ``r'   r  z.CriterionTest.test_cuda.<locals>.convert_dtype&  sS    #u||,zz|U3BB=QQC'QSQQQ
r)   r_  Tr  rP   g-C6:?r   rd  r  )r   r2  rp  rq  r   rG   r$  r  r   rA   r  ra  r  r   rs  r5   r  bfloat16r   r:  _backward_criterion)rm   r&  r   r  rw  
cpu_targetr{  r|  	gpu_input
gpu_targetr  r  r  r  r  s                 @r'   rN   zCriterionTest.test_cuda%  s7   	  5 5##$>??t112 %	{)I))+J)))4+@+@AJ)))4+@+@AJ &i=I++-1F1F1H*:u=
OOE"OOE" y)I
+JOO U^^44 OO-	!--/
-T--t/D/DE
"55j)Zdn5oJ"55j)Zdn5oJ!!*j/4U^^8T/TtZ^efty " { &99Iz:* : VM%99Iz:* : VM !!-/4U^^8T/TtZ^efty " {I%	{ %	{ %	{s   G3I  I	c                 &    | j                  dd      S r  r  r  s    r'   r  zCriterionTest._get_targetX  r  r)   c                 &    | j                  dd      S r  r  r  s    r'   rA   zCriterionTest.constructor_args[  r  r)   c                 &    | j                  dd      S )Nr  Fr  r  s    r'   r  zCriterionTest.extra_args_  s    }}\511r)   rk   )rr   rs   rt   r  r  unionr  r'  rN   r  r)  rA   r  r  r  s   @r'   r  r    s^     #66<<hZH; #\J1{f. 8 8 2 2r)   r  c                    t        j                  |t         j                  |d      }|Et        j                  |t         j                  |      |z  j                         j                         } ||      }t        j                  ||      }|j                  |       |j	                         }	|j                         j	                         j                         }
|j	                         } |	|
      }|j                  |       | j                  ||||d       | j                  |j                  j                  |
j                  j                  ||d       y )NT)r   r]  r  r\  )r]  Frd  )r5   r   r  r   r  rr  r  ru  r  r  r:  r  r  )r&  opr]  inp_dimsprecr<  rK  out1grad_input1op_bfp16rL  grad_input2out2s                r'   _test_bfloat16_opsr  d  s
   [[vUYZF**XU^^FKlZaacrrtf:D""47KMM+ {{}H]]_%%'668F&&(KFDMM+$4dN&++**FKK,<,<4d`efr)   c                    |s|j                  d        ||      }|s&t        j                  |      }|j                  |       |r.| j	                  |j                         |j                                |s|j                         D ]H  }|j                  s| j	                  |j                  t        j                  |j                               J | j	                  |j                  t        j                  |             y y )NT)
r  r5   	rand_liker  r:  rd   r  r  r  r  )r&  r  r  
check_size	inferencerA  gOr<   s           r'   _test_module_empty_inputr   w  s    4 
+C__S!Rchhj#((*5""$ 	HA%%affe.>.>qvv.FG	H 	chh(8(8(=>	 r)   c                       G d dt         j                         G fddt         j                        }         } |        }t        j                  ||      }|||fS )Nc                         e Zd Zd fdZ xZS ) _create_basic_net.<locals>.Layerc                     t         |           t        j                  t	        j
                  dd            | _        t        j                  t	        j                  dddd            | _	        y )NrW   rR   r3   r  )
r1  r  rv   r   r5   ro  layer_dummy_paramBufferr  layer_dummy_buf)rm   r6  s    r'   r  z)_create_basic_net.<locals>.Layer.__init__  sJ    G%'\\%++a2C%DD"#%99U[[Aq!-D#ED r)   r  Nrr   rs   rt   r  r  r  s   @r'   Layerr#    s    	F 	Fr)   r*  c                   $     e Zd Zd fdZ xZS )_create_basic_net.<locals>.Netc                     t         |                   | _        t        j                  t        j                  dd            | _        t        j                  t        j                  dddd            | _
        y )NrW   rR   r  r3   )r1  r  l1rv   r   r5   ro  dummy_paramr&  r  	dummy_buf)rm   r*  r6  s    r'   r  z'_create_basic_net.<locals>.Net.__init__  sR    GgDG!||EKK1,=>DYYu{{1aA'>?DNr)   r(  r)  )r6  r*  s   @r'   Netr,    s    	@ 	@r)   r1  )rv   rw   
Sequential)r1  lr5  r  r*  s       @r'   _create_basic_netr4    sR    F		 F@bii @ 	AA
aAa7Nr)   )r  F)Nr  )Nr  r  )Nr5  r  r  )r  rX  )r  )rX  r  )r3   r3   Nr  )r   r  )rX  rM   r  Fr  )rl   r  NrS  )abcr   r<  rp  copyr   	functoolsr   r   	itertoolsr   operatorr   r5   
torch.cudatorch.nnrv   torch.nn.functional
functionalr   r	   r"   $torch.testing._internal.common_utilsr
   r   r   r   r   r   r   r   #torch.testing._internal.common_cudar   r   torch.autograd.gradcheckr   r   torch.autogradr   torch.typesr   torch.backends.cudnntypingr   r   r   collections.abcr   r=  r  r(   r-   r   rc   module_testsrf   rx   r   r   r   r   r   r   r   r   r   r  r  r  r  r%  r6  r=  rD  rV  r[  rs  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r#  r)  r6  r?  rG  rU  r[  ri  ro  ru  r|  r  r  r  r  r  r  r  r  r  r!  r&  r*  r/  r6  r:  r<  r@  rD  rM  rX  rZ  rr  r   __annotations__criterion_testsr  regression_criterion_no_batch
reductionsr  r   regression_test_infor  !classification_criterion_no_batchr  r|  classification_cpp_parityr   r  r  classification_test_infor  r  r  r  r+  r  r  r  r  r   r4  rl   r)   r'   <module>rP     s       %        -C C C F K # (  ' ' $&&	'| 	 >Zll	 	'J:ll
 	ll	 	#Nll 	6ll	 	#+] ll	Y6x6
$$ $(&$$$$ "$ $ "$ $$$ $ $"$$$,$.$.$"$$$,$"$$$,$"$"$"$"$""$$$$$$$"$"$"$"$*$$$$$$$$$($,6"dN3 BF"(4 TZ=@, Z^RU%IP [a14
  @D &, 	 4.  ]b*0 "L %#$7DI $*$:6.0844 4- D) & D!  %
<jI 
1OD)6	{3'+S8!ll /0
1  
1I*9+6=/*8!ll /0
1" 	AI
 DFpqEGklGIlmACghPRijX3 DFde 	X2
  !KMcd1% !6 -u5A ,d39o 
   %
.56W6@/B 5*T8Y#6	{3'+S"'$)8 ,00t<  >AA$KJ##J/345 x> x>vJ" J"ZDH DHN$ $E8&
 E8Pv2& v2rg&? r)   