
    VhR}             &       2   U d dl Z d dlZd dlZd dl d dlmZ d dlZd dlmc mZ	 d dl
mZ d dlmZ ddlmZ g Zee   ed<   i Zeeef   ed	<   dd
Z	 	 	 ddZddZdeddfdZdededededdf
dZd Zd Zd Z e j>                  ee      Z dee!   fdZ"d Z#d Z$d Z%d Z& e ejN                  jP                  jR                  jT                  ejN                  jP                  jV                  jT                  ejN                  jP                  jX                  jT                  ejN                  jP                  jZ                  jT                  ejN                  jP                  j\                  jT                  ejN                  jP                  j^                  jT                  ejN                  jP                  j`                  jT                  gd      d        Z1 e ejN                  jd                  jf                  jT                  d      d        Z4 e ejN                  jP                  jj                  jT                  gd      d         Z6 e ejN                  jP                  jn                  jT                  d      d!        Z8  e ejN                  jP                  jn                  jr                  d"      e8        e ejN                  jP                  jt                  jT                  d#      d$        Z; e ejN                  jP                  jx                  jT                  d%      d&        Z= e ejN                  jP                  j|                  jT                  d'      d(        Z? e ejN                  jP                  j                  j                  d)      d*        ZB e ejN                  jP                  j                  jT                  d      d+        ZD e ejN                  jP                  j                  jT                  d,      d-        ZF  e ejN                  jP                  j                  jT                  d      e$        e ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  gd      d.        ZM  e ejN                  jP                  j                  jT                  d/      eM         e ejN                  jP                  j                  jT                  d0      eM         e ejN                  jP                  j                  j                  d1      eM        e ejN                  jP                  j                  jT                  d      d2        ZR e ejN                  jP                  j                  jT                  d3      d4        ZT e ejN                  jP                  j                  jT                  d5      d6        ZV e ejN                  jP                  j                  jT                  d7      d8        ZX e ejN                  jP                  j                  jT                  d9      d:        ZZ e ejN                  jP                  j                  j                  d;      d<        Z] e ejN                  jP                  j                  jT                  d=      d>        Z^ e ejN                  jP                  j                  j                  d?      d@        Za e ejN                  jP                  j                  jT                  dA      dB        Zc e ejN                  jP                  j                  jT                  dC      dD        Zd e ejN                  jP                  j                  jT                  dE      dF        Zf e ejN                  jP                  j                  j                  dG      dH        Zi e ejN                  jP                  j                  jX                  dI      dJ        Zk e ejN                  jP                  j                  jT                  dK      dL        Zm e ejN                  jP                  j                  jT                  dM      dN        Zo e ejN                  jP                  j                  jT                  dO      dP        Zq e ejN                  jP                  j                  jT                  dQ      dR        Zs e ejN                  jP                  j                  jT                  dS      dT        Zu e ejN                  jP                  j                  jT                  dU      dV        Zw e ejN                  jP                  j                  jT                  dW      dX        Zx e ejN                  jP                  j                  jT                  dY      dZ        Zy e ejN                  jP                  j                  j                  d[      d\        Z| e ejN                  jP                  j                  jT                  d]      d^        Z~ e ejN                  jP                  j                  jT                  d]      d_        Z e ejN                  jP                  j                  jT                  d`      da        Zdb Z e ejN                  jP                  j                  jT                  d=      dc        Z e ejN                  jP                  j                  j                  dd      de        Z e ejN                  jP                  j                  j                  df      dg        Z e ejN                  jP                  j                  jT                  dh      di        Z e ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  gdW      dj        Z e ejN                  jP                  j                  jT                  dk      dl        Z e ejN                  jP                  j"                  jT                  dm      dn        Z e ejN                  jP                  j&                  j                  do      dp        Z e ejN                  jP                  j*                  j                  dq      dr        Z e ejN                  jP                  j.                  jT                  ds       e ejN                  jP                  j0                  jT                  ds      dt               Z e ejN                  jP                  j2                  jT                  du      dv        Z e ejN                  jP                  j6                  jX                  dd      dw        Z e ejN                  jP                  j6                  jT                  d=      dx        Z e ejN                  jP                  j<                  j>                  dy      dz        Z e ejN                  jP                  j<                  jX                  d{      d|        Z e ejN                  jP                  jD                  j>                  dy      d}        Z e ejN                  jP                  jD                  jX                  d{      d~        Z e ejN                  jP                  jD                  jT                  ejN                  jP                  j<                  jT                  ejN                  jP                  jJ                  jT                  ejN                  jP                  jL                  jT                  gd      d        Z e ejN                  jP                  jL                  jX                  d{      d        Z e ejN                  jP                  jJ                  jX                  d{      d        Z e ejN                  jP                  jT                  jT                  dy      d        Z e ejN                  jP                  jX                  jT                  dy      d        Z e ejN                  jP                  j\                  jT                  dy      d        Z e ejN                  jP                  j`                  jT                  dy      d        Z e ejN                  jP                  jd                  jT                  d      d        Z e ejN                  jP                  jh                  jT                  dM      d        Z e ejN                  jP                  jl                  jT                  d      d        Z e ejN                  jP                  jp                  jT                  d      d        Z e ejN                  jP                  jt                  jT                  ejN                  jP                  jv                  jT                  gd      d        Z e ejN                  jP                  jD                  jT                  d      d        Z e ejN                  jP                  j|                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Z e ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  gd      d        Z e ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  ejN                  jP                  j                  jT                  gd      d        Z e ejN                  jP                  j                  j                  d      d        Z e ejN                  jP                  j                  j                  d      d        Z e ejN                  jP                  j                  j                  d      d        Z e ejN                  jP                  j                  jT                  d      d        Zd dlmZmZ d dlmZ  ej                  e      	 	 ddej                  dej                  dej                  de!dededeeef   dededeej                  ej                  f   fd       Z ej                  e      	 	 ddej                  dej                  dej                  dej                  dej                  dej                  dej                  dee!ef   dedededeeef   dededeej                  ej                  ej                  eeej                     df   f   fd       Z e ejN                  jP                  j                  jT                  d      d        Zej                  j                  dd      5 Z(e(j                  dedë       e(j                  dedī       e(j                  dedū       ddd       y# 1 sw Y   yxY w)    N)*)Optional)normalize_function)#jagged_scaled_dot_product_attention   NestedTensor__all__JAGGED_OPS_TABLEc                     ddl m} t        |t        t        f      r? t        |       fd|D              } t        |      t        j                  |            S |r	 | |      }|dk\  r| k  sJ |dk(  rdz
  S |dz
  S )Nr   canonicalize_dimsc              3   8   K   | ]  }t        |        y wN_outer_to_inner_dim).0dndim
ragged_dims     J/home/dcms/DCMS/lib/python3.12/site-packages/torch/nested/_internal/ops.py	<genexpr>z&_outer_to_inner_dim.<locals>.<genexpr>   s     Q.tQ
CQ   r   )torch._prims_commonr   
isinstancetuplelisttypedictfromkeys)r   dimr   canonicalizer   outputs   ` `   r   r   r      s    5#t}%cQSQQtF|DMM&122c*!8d
"" !AX:>2372    c                     ddl m}  || |      }||k(  r|st        | d      |dk(  r|st        | d      |rt        | ||      n|}	|r	|dk(  }
|	|
fS |	S )Nr   r   z0(): not supported for NestedTensor on ragged dimz+(): not supported for NestedTensor on dim=0)r   r   RuntimeErrorr   )r   r!   r   op_nameconvert_to_inner_dimallow_ragged_dimallow_batch_dimr   wrappedretoperating_on_batchs              r   _wrap_jagged_dimr.   '   s     6c*G*%5gY&VWXX	AogY&QRSS   	D':6 
  %\'((Jr$   c                 .    ddl m} t        |t        t        f      sJ dt        |              |D cg c]  } | |       }}d|v }|v }t        fd|D              }	t        t        j                   fd|D                    }
|
|||	fS c c}w )z
    For NestedTensor operators,
    wraps dimensions to non-negative values,
    and returns metadata related to reduction dimension(s).
    r   r   z<_wrap_jagged_dims(): cannot iterate over dimensions of type c              3   6   K   | ]  }|d k7  xr |k7    yw)r   N )r   r   
ragged_idxs     r   r   z$_wrap_jagged_dims.<locals>.<genexpr>V   s!     PaqAv9!z/9Ps   c              3   8   K   | ]  }t        |        y wr   r   )r   r   r   r2   s     r   r   z$_wrap_jagged_dims.<locals>.<genexpr>Z   s     U1)$:>Ur   )	r   r   r   r   r   r   anyr   r    )r   dimsr'   r2   r   r   wrapped_dimsoperate_on_batchoperate_on_raggedoperate_on_non_batchouter_to_inner_dims   `  `       r   _wrap_jagged_dimsr;   D   s     6udm S	Ed4j\RS 
 -1'($"L  L("l2P<PP UUU /1BDXXXs   B
schema_strreturnc                 :   | j                  d      }|D cg c]  }|j                  d       c}j                  d      }t        |      |z
  }|d   dk(  r|d d }n\t        |      |k\  rt        |      t        |      k  s7t	        d|j
                   d|  d| d	t        |       d
t        |       d      d d d d d}t        |      D ]  \  }	}
|
j                  d      \  }}|j                  d      }|r|d d n|}||j                         vrt        d|       |	t        |      k\  r!|st	        d|j
                   d|  d|       ||   |ffd	} |||	         rdddddd}t	        d|j
                   d|  d| d||           y c c}w )N, ?T...NestedTensor (z): expected at least z arguments and at most z arguments, but got: z
 argumentsc                 \    t        | t        j                        xr t        | t               S r   )r   torchTensorr	   xs    r   <lambda>zcheck_schema.<locals>.<lambda>q   s     z!U\\2V:a;V7V r$   c                 d    t        | t              xr | j                  d u xr | j                  dk(  S Nr   )r   r	   _lengths_ragged_idxrH   s    r   rJ   zcheck_schema.<locals>.<lambda>r   s2    
1l3 JJ$MMQ r$   c                 "    t        | t              S r   r   r	   rH   s    r   rJ   zcheck_schema.<locals>.<lambda>u   s    J|
 r$   c                      y)NTr1   rH   s    r   rJ   zcheck_schema.<locals>.<lambda>x       r$   )tjtjt_allr4   z: zUnknown arg type: z) missing required argument: c                 4    |r| d u xs  |       S  |       S r   r1   )rI   is_optional	_check_fns     r   check_fnzcheck_schema.<locals>.check_fn   s#    Dy0IaL0 |#r$   tensorzoptional tensorz%contiguous jagged layout NestedTensorzjagged layout NestedTensorz
<any type>)rS   zt?rT   rU   r4   z): expected z	 to be a )	splitendswithcountlen
ValueError__name__	enumeratekeysAssertionError)r<   funcargskwargsnamed_arg_typesrI   num_optional_argsmin_argsarg_type_check_fnsinamed_arg_typenamearg_typerW   normalized_arg_typerY   type_to_descrX   s                    @r   check_schemarq   `   s    &&t,O2ABQCBHHN?#&77H re#)#2.D	X%#d)s?7K*Ka
|;PQYPZ [)),_)=(>>St9+Z)  W
 	 '7 #>'--d3h'',/:hsm&8&=&=&?? #56I5J!KLLD	> #DMM?!J< @2269  &':;	$/ 	$ Q '=6#L a
|<vY)*, A#1 Cs   Faa_namebb_namec           	          |j                   |j                     |j                   |j                     k7  rt        d| j                   d| d| d      y )NrC   z: expected  and z' to have the same exact offsets tensor.)_sizerN   r&   r`   )rd   rr   rs   rt   ru   s        r   check_ragged_dim_samery      sZ     	wwq}}!77DMM?+fXU6( K) )
 	
 8r$   c                     | j                   dz   }| j                  d | }|d | }t        |      t        |      k(  xr t        d t	        ||      D              S )Nr   c              3   :   K   | ]  \  }}||k(  xs |d k(    yw)rA   Nr1   )r   nsss      r   r   z%raggedness_matches.<locals>.<genexpr>   s$     H52qB!GqBwHs   )rN   rx   r^   allzip)ntsizeend	nt_raggedsize_raggeds        r   raggedness_matchesr      sZ    
..1
C#It*Ky>S-- HC	;,GHHr$   c                     | j                         dkD  rI| j                  d   dk(  r7| j                  d      } | j                         dkD  r| j                  d   dk(  r7| S Nr   r   )r!   shapesqueeze)rS   s    r   squeeze_leading_onesr      sN    & %%'A+!''!*/IIaL %%'A+!''!*/Hr$   c                 f     t        t              sgt         t              s g  fd}|S )Nc                 H     D ]  } fd}D ]  } ||      ||<      S )Nc                       fd}|S )Nc                  <    t        g| i |  g| i |S r   )rq   )re   rf   aten_oprd   r<   s     r   innerz@register_func.<locals>.wrapper.<locals>.get_inner.<locals>.inner   s,     TCDCFC9$9&99r$   r1   )r   r   rd   r<   s   ` r   	get_innerz1register_func.<locals>.wrapper.<locals>.get_inner   s    : r$   r1   )rd   r   r   tableaten_opsr<   tabless   `   r   wrapperzregister_func.<locals>.wrapper   s=     
	4G   4!*7!3g4
	4 r$   )r   r   )r   r   r<   r   s   ``` r   register_funcr      s1    h%:fd# Nr$   c                    t         j                  | d       }||S t        j                  j                  | j
                  v rAddlm} |D ]%  } ||      st        d| j                   d|        t        d |D              }|dk(  rg }| j                  j                  D ]d  }t        |j                  t        j                        r |j!                  |j"                   d        n |j!                  |j"                   d       f |j!                  d	       d
j%                  |      }t'        || g|i | t)        j*                  t,        |       S |dk(  r*t'        d| g|i | t)        j*                  t.        |       S y )Nr   is_nested_intrC   z: invalid argument c              3   P   K   | ]  }t        |t        j                           y wr   )r   rF   rG   )r   rI   s     r   r   z lookup_jagged.<locals>.<genexpr>   s     HajELL9Hs   $&r   z: jt_allz: anyrB   r?      zlhs: any, rhs: any, ...)r   getrF   Tag	pointwisetags%torch.fx.experimental.symbolic_shapesr   r&   r`   sum_schema	argumentsr   r   
TensorTypeappendrm   joinrq   	functoolspartialjagged_unary_pointwisejagged_binary_pointwise)	rd   re   rf   dispatch_funcr   argnum_tensor_argsschema_partscheck_schema_strs	            r   lookup_jaggedr      su   $((t4M  yydii'G
  	CS!"#DMM?2EcUK 	 H4HHa L||-- <chh(8(89 ''388*H(=> ''388*E(:;< &#yy6)4A$A&A$$%;TBB!2DJ4J6J$$%<dCCr$   c                 t    | j                         | j                         | j                  | j                  d}|S )Noffsetslengths_metadata_cacherN   r   )r   rf   s     r   extract_kwargsr     s2    ;;=;;=..	F Mr$   c                 p    t        d |D              t         | fd|D        i |fi t              S )Nc              3   B   K   | ]  }t        |t              s|  y wr   rP   r   r   s     r   r   z)jagged_unary_pointwise.<locals>.<genexpr>  s     Dsjl&CsD   c              3   B   K   | ]  }|u r|j                   n|  y wr   )_values)r   r   njts     r   r   z)jagged_unary_pointwise.<locals>.<genexpr>  s      BccSjs{{c1Bs   nextr	   r   )rd   re   rf   r   s      @r   r   r     s@    
DdD
DCBTBMfM

 r$   c                 Z   |d   |d   }}t        |t              st        |t              sJ d}t        |t              rt        |t              rt        ||j                        r7t         | |j                  |j                  g|dd  i |fi t        |      S t        |j                  | j                  |j                  |j                              t        |t              }|rt        |      n
t        |      }|r||fn||f\  }}	|	j                         |j                         kD  rt        d      t        |	      }
|j                         |
j                         dz   k\  r;|r|j                  |
fn|
|j                  f\  }}t         | ||g|dd  i |fi |S |j                         |j                         k(  r*|j                  d   |j                  d   k7  r:t        |j                  | j                  |j                  |j                              ddlm} |j                  }|j                   }|}|j                  j                  |j"                  dz
     }||}|	j%                  t'        |	j                        D cg c]  \  }}||j"                  k7  r|n| c}}      }	 ||	|j(                  |j"                  |||      }|r||fn||f\  }} | ||g|dd  i |S t        |j                  | j                  |j                  |j                              c c}}w )Nr   r   zHcannot call binary pointwise function {} with inputs of shapes {} and {}r   z+NYI: broadcasting NT with T with larger dimnested_from_paddedr   r2   sum_S
min_seqlen
max_seqlen)r   r	   r   rx   r   r   r&   formatr`   r!   NotImplementedErrorr   r   nested_tensorr   _maybe_min_seqlen_maybe_max_seqlenrN   expandra   _offsets)rd   re   rf   rr   rt   mismatch_error_msga_is_ntextracted_kwargsr   rS   
t_squeezedlhsrhsr   r   r   padded_max_Stotal_Lrk   rI   t_as_nts                        r   r   r   $  s   7DGqAa&*Q*EEE 	S  !\"z!\'B a)QYY		?DH??CQRSCT  -44T]]AGGQWWUVVL)G,3~a(9J QFAq6EBuuw!"OPP%a(J	vvx:>>#a''/6BJJ
+Z<TSDc?DH??TCSTT 	uuw!%%' 771:#"))$--!''J  	6 ))
))
!**""2>>A#56"L HHDMaggDVWDAq!r~~%Q<7W
 %KK~~!!
 %,B='2SC2tABx2622 )00Q
RR# Xs   	L'
c                 ^   | t         j                  j                  j                  u rt	        |i |S | j
                  dk(  r! | |d   j                  g|dd  i | |d   S | j
                  dk(  rdd}t        |||d      \  }}|j                  d      }t        |j                         |d	   |j                  dd
      }t        |j                         |d   |j                  dd
      }||k(  r|S t        j                  t        j                  |j                   ||dz          }	g |j                   d | |	|j                   |dz   d  }
 |j"                  |
 S | j
                  dk(  rdd}t        |||d      \  }}|j                  d      }|j                  d      }|j                         |j                  z
  dz
  }t%        |      |kD  rt'        d      t         j                  j)                         5   | |i |cd d d        S t+        |       # 1 sw Y   t+        |       xY w)Napply_r   r   flattenc                      y r   r1   )input	start_dimend_dims      r   _flatten_sigz+jagged_torch_function.<locals>._flatten_sig      r$   Tre   rf   normalize_to_only_use_kwargsr   r   F)r(   r   rms_normc                      y r   r1   )r   normalized_shapeweightepss       r   _rms_norm_sigz,jagged_torch_function.<locals>._rms_norm_sig  r   r$   r   zNrms_norm(): Normalization over the ragged dim not supported for nested tensors)r   rA   NN)rF   _C_nnscaled_dot_product_attentionr   r`   r   r   popr.   r!   rN   r   reduceoperatormulr   reshaper^   r_   DisableTorchFunctionSubclassr   )rd   re   rf   r   _
new_kwargsinpr   r   product	new_shaper   r   max_normalizables                 r   jagged_torch_functionr   r  s:    uxx||8882DCFCC}} T!W__2tABx262Aw }}	!	 +tFQU
: nnW% %GGI{#OO!&
	 #GGIy!OO!&
 J""8<<9wQR{1STPcii
+PWPsyy17OP	s{{I&& }}
"	 +VRV
: nnW%%>>*<= 779s6: #33`  XX224 	)((	) 	) d
##	) d
##s   <HH,zself: jt_allc                 8   | t         j                  j                  j                  j                  k(  ry| t         j                  j                  j
                  j                  k(  r|d   j                  S | t         j                  j                  j                  j                  k(  rt        |d   j                        S | t         j                  j                  j                  j                  t         j                  j                  j                  j                  fv rs|d   j                  Gt        t        |d   j                        t        j                  |d   j                  dd        z        S |d   j                   j                         S | t         j                  j                  j"                  j                  k(  r|d   j$                  S | t         j                  j                  j&                  j                  k(  r|d   j                   j)                         S y )NFr   r   )rF   opsatenis_non_overlapping_and_densedefaultsym_sizerx   r!   r^   	sym_numelnumelrM   intr   mathprodr   
sym_stride_stridessym_storage_offsetstorage_offsetrd   re   rf   s      r   tensor_attr_supported_getterr	    ss    uyy~~::BBBuyy~~&&...Aw}}uyy~~!!)))47==!!		((00%))..2F2F2N2NOO7's47++,tyyaqr9J/KKLLAw$$&&uyy~~((000Awuyy~~00888Aw--// 9r$   c                 "    t         j                  S r   )rF   jaggedr  s      r   prim_layout_defaultr    s    <<r$   c                 |    | t         j                  j                  j                  j                  k(  rt        d      y )NznNestedTensor does not support directly calling torch.ops.aten.size; please use `nested_tensor.size()` instead.)rF   r   r   r   r   r&   r  s      r   tensor_attr_unsupported_getterr    s7    
 uyy~~""***9
 	
 +r$   c                    ddl m} t        | ||d      \  }}|j                  d      }|j	                         y|j                  dt        j                        |d<   |d   t        j                  k(  ry ||j                  fi |S )Nr   )is_contiguous_for_memory_formatTr   r   Fmemory_format)
r   r  r   r   r   r   rF   contiguous_formatpreserve_formatr   )rd   re   rf   r  r   r   r   s          r   is_contiguous_generalr    s    C&4TMAz ..
!C {{} ",..00#J /"e&;&;;*3;;E*EEr$   z!self: jt_all, memory_format: any?z"input: jt_all, memory_format: any?c                 N   t        | ||d      \  }}|j                  d      }t        |      }|j                  O|d   t        j
                  k(  r9ddlm} |j                  dk(  sJ d        ||j                         d       \  }}|S t         | |j                  fi |fi |S )	NTr   r   r  r   )jagged_from_listz;NJT with ragged_idx != 1 not supported for contiguous clone)r   )r   r   r   rM   rF   r  r   r  rN   unbindr	   r   )	rd   re   rf   r   r   r   new_metar  contigs	            r   clone_defaultr    s     '4TMAz ..
!Cc"H
||o&%*A*AA 8 1$MLM$(tDIFAMS[[7J7D8DDr$   zinput: jt, weight: t, bias: t?c                     t        | ||d      \  }}|j                  d      }t         | |j                  fi |fi t	        |      S NTr   r   r   r   r	   r   r   rd   re   rf   r   r   r   s         r   linear_defaultr  #  L    &4TMAz ..
!CS[[7J7O>#;NOOr$   z6self: jt, grad_output: jt, weight: t, output_mask: anyc                 4   t        | ||d      \  }}|j                  d      }|j                  d      }|j                  d      }|j                  d      }d\  }	}
}t        | |d|d       |d	   r3t        t	        j
                  |j                  |      fi t        |      }	|d
   rz|j                  j                  d|j                  d	            }|j                  j                  d|j                  d
            }t	        j
                  |j                         |      }
|d   rmt        t        |j                  j                  d
z
              }|dk(  r|j                  j                         }n"t	        j                  |j                  |d      }|	|
|fS )NTr   r   grad_outputr   output_mask)NNNselfr   r   rA   r   r1   F)keepdim)r   r   ry   r	   rF   matmulr   r   r   r   rS   r   ranger   cloner   )rd   re   rf   r   r   r   r"  r   r#  dsdwdbgrad_2dinput_2dreduce_dimss                  r   linear_backward_defaultr/  .  sj   
 '4TMAz ..
!C../K^^H%F../K!JBB$V[-H1~LL,,f5
9G9T
 1~ %%--b&++a.A;;&&r6;;q>:\\'))+x01~ E+"5"5":":Q">?@"$$**,B;..UKBB<r$   zinput: jt_all, dtype: anyc                     t        | ||d      \  }}|j                  d      }t         | |j                  fi |fi t	        |      S r  r  r  s         r   to_dtyper1  W  r   r$   c                 v   ddl m} t        | ||d      \  }}|j                  d      }|j                  d        | |j                  fi |}|j
                  j                  |j                        }d }	|j                  &|j                  j                  |j                        }	dd	l	m
}
 dd
lm}m} |j                  |j
                  n|j                  }|	|n|	}t        ||
|f      r" ||      } ||      }|j                  |_        n||   ||<   t!        |      }||d<   |	|d<   t#        |fi |}|S )Nr   _tensor_symint_registryTr   r   layoutdevicer   
FakeTensorFunctionalTensormb_unwrap_functional_tensorr   r   )r   r4  r   r   r   r   tor7  rM   torch._subclasses.fake_tensorr9  #torch._subclasses.functional_tensorr;  r<  r   nested_int_memor   r	   )rd   re   rf   r4  r   r   r   
new_valuesnew_offsetsnew_lengthsr9  r;  r<  ragged_source	new_thingtgtsrc
inp_kwargsr#   s                      r   to_copy_defaultrI  b  s0   6&4TMAz ..
!CNN8ckk0Z0J,,//):):/;KK
||llooZ->->o?8
 %(LL$8CLLcllM*2I)j*:;<))4)-8!11-D]-S	*$J'Jy'Jy*3
3FMr$   z-self: jt_all, src: jt_all, non_blocking: any?c                 @   t        | ||d      \  }}|j                  d      }|j                  d      }|j                  |j                  k7  r|j                         }|D cg c]  }|j                   }	}|j                         }
|
D cg c]  }|j                   }}|	|k7  r%t        d|j                   d|j                         t        ||
      D ]  \  }}|j                  |        |j                  j                  |j                         |S c c}w c c}w )NTr   r   rG  z<copy_(): expected compatible input and src shapes, but got: rw   )	r   r   rx   r  r   r&   r   copy_r   )rd   re   rf   r   r   r   rG  	inp_compscinp_comp_shapes	src_compssrc_comp_shapesinp_compsrc_comps                 r   copy_defaultrS    s    '4TMAz ..
!C
..
C
yyCII JJL	,56q17766JJL	,56q17766o-N99+U399+/  #&i"; 	%HhNN8$	% KKckk"J! 76s   "DDc                    t        | ||d      \  }}|j                  d      }t        j                  |d<    | |j                  fi |}|j
                  j                  |j                        }d }|j                  &|j                  j                  |j                        }t        |      }	d|	v r||	d<   d|	v r||	d<   |j                  |j                  k7  rvdd	l
m}
 dd
lm}m} ddlm} |j                  |j
                  n|j                  }||n|}t#        ||
|f      r" ||      } ||      }|j$                  |_        n||   ||<   t'        |fi |	S )NTr   r   r5  r6  r   r   r   r8  r:  r   r3  )r   r   rF   stridedr   r   r=  r7  rM   r   r>  r9  r?  r;  r<  r   r4  r   r@  r	   )rd   re   rf   r   r   r   rA  rB  rC  output_kwargsr9  r;  r<  r4  rD  rE  rF  rG  s                     r   like_factory_defaultrW    sV    '4TMAz ..
!C
 !==Jxckk0Z0J,,//):):/;KK
||llooZ->->o?"3'MM!#.i M!#.i 
zzZ&&& 	=	

 	;(+(<#,,#.#6KK	i*.>!?@-i8C-m<C"%"5"5C1H1W#I.
4m44r$   zself: jt_all, fill_value: anyzself: jt_all, high: anyz!self: jt_all, low: any, high: anyc                 p    t        | ||d      \  }}|j                  d      } | |j                         |S r  r   r   r   r  s         r   zero__defaultrZ    s:    &4TMAz ..
!CJr$   z*self: jt_all, dim: any, half_to_float: anyc           
      H   t        | ||d      \  }}t        |d   t              rt        d      |j	                  d      }t        |j                         |d   fd|j                        \  |d<   }}}|rt        d      |r|j                  dkD  rt        d	      |r|j                  t        d
      |d   d   |d<   |r:t        j                  j                  j                  t        j                  j                  j                  |j                   j#                  |j                   j$                  d   d      |j&                  g|j(                  gt+        d            |j                        }	t        j                  j                  j-                  |	|j&                  g|j                   j$                  d         j"                  dg|j                   j$                  dd   }
t/        |
fi t1        |      S t/         | |j                   fi |fi t1        |      S )NTr   r!   zHsoftmax(): not supported for dimensions of type 'tuple' for NestedTensorr   softmaxzRsoftmax(): not supported when reducing across the batch dimension for NestedTensorr   zesoftmax(): not supported when reducing along the ragged dimension for ragged_idx > 1 for NestedTensorzksoftmax(): not supported where lengths is not None if reducing across the ragged dimension for NestedTensorr   rA   z-inf)max_lengthspadding_valuer!   r   )r   r   r   r&   r   r;   r!   rN   rM   rF   nn
functionalr\  r   r   _jagged_to_padded_dense_forwardr   r   r   r   _max_seqlenfloat_padded_dense_to_jagged_forwardr	   r   )rd   re   rf   r   r   r   reduce_on_batchreduce_on_ragged_reduce_on_non_batchpadded_softmax_valuessoftmax_valuess              r   _softmax_defaultrl    s)    '4TMAz *U#U+V
 	
 ..
!C 			E			5 `
 	
 COOa/s
 	
 CLL4I
 	

 #5)	Ju  % 3 3 ; ;IINN::##KK%%a("  __-#Fm ;   !< 
!
 GG!\\NKK%% H 
 '
 ""12&
 NBnS.ABBS[[7J7O>#;NOOr$   z7grad_output: jt, output: jt, dim: any, input_dtype: anyc                     t        | ||d      \  }}|j                  d      }|j                  d      }t         | |j                  |j                  fi |fi t	        |      S )NTr   r"  r#   r  )rd   re   rf   r   r   grad_outr#   s          r   _softmax_backwardro  L  sl    
 '4TMAz ~~m,H^^H%FXv~~<<@Nx@X r$   z!self: jt, float: any, train: any?c                     t        | ||d      \  }}|j                  d      } | |j                  fi |\  }}t        |fi t	        |      t        |fi t	        |      fS r  r   r   r   r	   r   )rd   re   rf   r   r   r   out1out2s           r   native_dropout_defaultrt  [  sp     '4TMAz ..
!Cckk0Z0JD$T1^C01T1^C01 r$   z%grad_output: jt, mask: jt, scale: anyc                     t        | ||d      \  }}|j                  d      }|j                  d      }t         | |j                  |j                  fi |fi t	        |      S )NTr   r"  maskr  )rd   re   rf   r   r   r"  rv  s          r   native_dropout_backward_defaultrw  k  sl    
 '4TMAz ../K>>&!D[  $,,=*=

% r$   z2self: jt_all, dim: any, keepdim: any?, dtype: any?c                 $    t        | ddg|i |S )Nr  r   _apply_reductionr  s      r   prod_dim_intr{  {  s    
 D&!=d=f==r$   zself: jt_all, dtype: any?c                 n    t        | ||d      \  }}|j                  d      } | |j                  fi |S r  rY  r  s         r   prod_defaultr}    =    &4TMAz ..
!C*z**r$   z$self: jt, split_size: any, dim: any?c                     t        | ||d      \  }}|j                  d      t        j                         |d   j                  d      |d<   t        fd | j                  fi |D              S )NTr   r   r!   r[   c              3   J   K   | ]  }t        dd |it                yw)valuesNr1   )r	   r   )r   rI   r   s     r   r   zsplit_tensor.<locals>.<genexpr>  s+       	5A5!45s    #)r   r   r.   r!   rN   r   r   r  s        @r   split_tensorr    s}     '4TMAz ..
!C(	:e$coowJu  ckk0Z0  r$   z%self: jt, split_sizes: any, dim: any?c           
         t        | ||d      \  }}|j                  d      }t        |j                         |d   |j                  d      |d<    | |j
                  fi |D cg c]  }t        dd|it        |       c}S c c}w )NTr   r   r!   split_with_sizesr  r1   r   r   r.   r!   rN   r   r	   r   )rd   re   rf   r   r   r   rI   s          r   split_with_sizes_defaultr    s     '4TMAz ..
!C(	:e$coo7IJu ckk0Z0 	5A5!45  s   %Bz+self: jt, dim: any, start: any, length: anyc                     t        | ||d      \  }}|j                  d      }t        |j                         |d   |j                  d      } | |j
                  ||d   |d         }t        |fi t        |      S )	NTr   r   r!   narrowstartlengthr!   r  r  r  )rd   re   rf   r   r   r   r!   r  s           r   r  r    s     '4TMAz ..
!C
3779j&7(
SC!(#	F 6."566r$   z self: jt, chunks: any, dim: any?c           
      t   t        | ||d      \  }}|j                  d      }t        |j                         |d   |j                  dd      \  |d<   }|r|d   }|j
                  j                         }|j                  |      }	|	D 
cg c]  }
t        j                  |
d	       }}
|D 
cg c]  }
t        j                  |
d
d       }}
|D cg c]  }||j                  d }}|	D 
cg c]   }
|
j                         j                         " }}
|j                  j                  |      }t!        dt#        |            D cg c]  }t%        dd||   i||    c}S  | |j                  fi |D 
cg c]  }
t%        dd|
it'        |       c}
S c c}
w c c}
w c c}w c c}
w c c}w c c}
w )NTr   r   r!   chunkr*   chunksr   r_  r   r   )value)r   rN   r  r1   )r   r   r.   r!   rN   r   diffr  rF   cumsumFpadr   itemr   r[   r'  r^   r	   r   )rd   re   rf   r   r   r   r-   r  r   chunked_lengthsrI   chunked_offsetsper_offsetsnested_kwargssplit_sizeschunk_valuesrk   s                    r   chunk_defaultr    s   &4TMAz ..
!C,<	:e$coowPT-)Ju) H% ,,##%!--/;JKa5<<q1KK>MN155F!4NN  /
 $COOD
 
 0??!quuw||~??{{((5 1c,/0
 FQFM!4DF
 	
 #++44
 99^C%89
 	
% LN
 @



s$   F0F!F&1%F+
F0<F5zself: jt_all, dim: any?c           
         t        | ||d      \  }}|d   }|dk7  rt        d      |j                  d      }|j                         |j	                         }|j                         }|j                  ddt        t           dt        t        t              ffd	}	|A|j                         j                         }
 |	|
       t        j                  |
d
z
        S dk  rt        d      |j                         }
|j                         } |	|
|       t        |j                  d         D cg c]$  }t        j                   d
z
  ||   |
|         & c}S c c}w )NTr   r!   r   z2unbind(): only supported for NestedTensor on dim=0r   rM   r   c                    d}j                   dz
     }t        t        |             D ]c  }t        j                  | |          t        j
                  | |   |k         || |   z  }|At        j
                  ||   | |   z   |k  d        e t        j
                  ||k         |Mt        t        |            D ]5  }t        j                  ||          t        j
                  ||   |k         7 y y )Nr   r   c                       y)NzMunbind(): nested tensor offsets and lengths do not match ragged_idx dimensionr1   r1   r$   r   rJ   z2unbind_int.<locals>._torch_check.<locals>.<lambda>  rR   r$   )r   r'  r^   rF   _check_is_size_check)rM   r   lengths_sumragged_dim_sizerk   r2   r  s        r   _torch_checkz unbind_int.<locals>._torch_check  s      ,,zA~6s8}% 		A  !-LL!788A;&K#QK(1+-@k		 	[O343x=) =$$Xa[1Xa[O;<=  r$   r   r_  zAunbind(): nested tensor ragged_idx out of bounds (should be >= 1)r  r   )r   r&   r   r  r   r   rN   r   r  r   r  tolistrF   r[   r'  r   r  )rd   re   rf   r   r   r!   r   r   r   r  lengths_scalarsoffsets_scalarsrk   r2   r  s                @@r   
unbind_intr    s]    '4TMAz U
C
axOPP
..
!CZZ\FkkmGkkmGJ=tCy =HT#Y4G =: !,,.//1_%{{6?aIIQO
 	
 nn&Onn&O/2 w}}Q'(  	a!!$"1%		
  s   6)E"zself: jt, dim: anyc                     t        | ||d      \  }}|j                  d      }|j                  }t        t	        |j
                        |d   |j                  d      |d<   t         | |fi |fi t        |      S )NTr   r   r!   r   )	r   r   r   r.   r^   rx   rN   r	   r   )rd   re   rf   r   r   r   r  s          r   squeeze_dimr  <  s|    &4TMAz ..
!C[[F(CII
5)3??IJu V2z2JnS6IJJr$   zself: jt_all, dim: anyc                 P   t        | ||d      \  }}|j                  d      }|j                  }|d   }t        t	        |j
                        dz   ||j                  dd      |d<   t        |      }|d   |j                  dz
  k  r|dxx   dz  cc<   t         | |fi |fi |S )	NTr   r   r!   r   	unsqueeze)r)   rN   )	r   r   r   r.   r^   rx   rN   r   r	   )	rd   re   rf   r   r   r   r  r!   rV  s	            r   unsqueeze_defaultr  K  s    &4TMAz ..
!C[[F U
C(CIIC+PTJu
 #3'M%COOa//m$)$V2z2DmDDr$   ztensors: any, dim: anyc                    t        | ||d      \  }}|j                  d      }|D cg c]  }|j                  s| }}t        |      dkD  sJ |d   }|D cg c]!  }|j                  r|n|j	                  |      # }}|d   }	t        t        |j                        |	|j                  d      |d<   t         | |D cg c]  }|j                   c}fi |fi t        |d         S c c}w c c}w c c}w )NTr   tensorsr   r!   cat)r   r   	is_nestedr^   	expand_asr.   r   rN   r	   r   r   )
rd   re   rf   r   r   r  rS   nestedfirstr!   s
             r   cat_defaultr  b  s   &4TMAz nnY'G !0AAKKa0F0v;??1IEAHIAAKKqQ[[%77IGI U
C(EKK#u00%Ju )Aaii)8Z8<J7ST:<V  1 J *s   C+C+&C0?C5zself: any, other: anyc                     t         ||d      \  }}|j                  d      }|j                  d      } fd} fd}|j                  r|j                  s|j                         dk\  rJ|j                         |j                         k(  r)|j                  |j                         dz
  k  r	 |||      S |j                         d	k(  rX|j                         |j                         kD  r6|j                  |j                         dz
  k  rt          |j                  |fi |fi t        |      S |j                  s|j                  r|j                         dk\  r9|j                         |j                         k(  r|j                  d	k\  r	 |||      S |j                         d	k(  rv|j                         |j                         kD  rT|j                  d	k\  rDt          ||j                  fi |fi t        |      S |j                  r|j                  r|j                         dkD  rY|j                         dkD  rFt        ||j                        r0t          |j                  |j                        fi t        |      S |j                         dk(  r|j                         dk(  rq|j                  d	k(  rb|j                  dk(  rS|j                  |j                        |j                  |j                        k(  rt        j                   |||            S t        d
|j                   d|j                         )NTr   r   otherc                     t        | j                         |j                               D cg c]  \  }} ||       c}}S c c}}w r   )r   r  )rr   rt   a_compb_comprd   s       r   _unbind_implz$matmul_default.<locals>._unbind_impl  s>    9<QXXZ9T
%5ffD 
 	
 
s   Ac                    | j                   r| }n|}ddlm} |j                  }|j                  }|}|j
                  j                  |j                  dz
     }||}g |j                  d |j                   ||j                  |j                  dz   d  }|j                  d|      }	| j                   r
 |	|      }
n	 | |	      }
 ||
|j                  |j                  |||      S )Nr   r           output_sizer   )
r  r   r   r   r   r   r   rN   to_padded_tensorr   )rr   rt   r   r   r   r   r   r   padded_shape	padded_ntpadded_trd   s              r   _padded_implz$matmul_default.<locals>._padded_impl  s    ;;BB5))
))
!**""2>>A#56"L
XX&'

 XXbnnq(*+

 '''F	;;Iq)HAy)H!KK~~!!
 	
r$      r   r   z1matmul(): not supported between inputs of shapes rw   )r   r   r  r!   rN   r	   r   r   r   rx   r   rF   stackr&   r   )	rd   re   rf   r   r   r   r  r  r  s	   `        r   matmul_defaultr  {  s   &4TMAz ..
!CNN7#E

!
J }}U__ GGIN	UYY[(#'')a-/  U++ IIK1	EIIK'#'')a-/S[[%6:6:H:M  ]]u99;!		swwy 8U=N=NRS=SU++
 WWY!^		cggi 7E<M<MQR<RS%--6:6:H:O 
 
5?? 779q=UYY[1_1CC1US[[%-- @XNSVDWXX GGIN		q 1$!!Q&)UZZ8I8I-JJ ;;|C788

;CII;eEKK=Y r$   zself: jt_all, mat2: anyc                 T   t        | ||d      \  }}|j                  d      }|j                  d      }|j                         dk7  rt        d      |j                         dk7  rt        d      t	        t
        j                  j                  j                  j                  ||      S )NTr   r   mat2r  zbmm(): input must be 3Dzbmm(): mat2 must be 3D)
r   r   r!   r_   r  rF   r   r   r&  r   rd   re   rf   r   r   r   r  s          r   bmm_defaultr    s    &4TMAz ..
!CNN6"E
wwyA~233yy{a122%))..//77eDDr$   z'self: jt_all, size: any, implicit: any?c                    t        | ||d      \  }}|j                  d      }|d   }d|vs|j                  d      rJ t        ||      st        d|j                   d|       t        d|j                               D cg c]  }||j                  k(  rd	n||    }}t         | |j                  |      fi t        |      S c c}w )
NTr   r   r   implicitzexpand(): cannot expand shape z -> r   rA   )r   r   r   r&   rx   r'  r!   rN   r	   r   r   )	rd   re   rf   r   r   r   r   r   
expand_args	            r   expand_defaultr    s     '4TMAz ..
!CfDj(*..2LMMc4(;CII;d4&QRRCHCGGICVWaS__,"$q'9WJWS[[*5M9LMM Xs   Czself: t, other: jtc                     t        | ||d      \  }}|j                  d      }|j                  d      }t         | ||j                        fi t	        |      S )NTr   r   r  r  r  s          r   expand_as_defaultr    sW    &4TMAz ..
!CNN7#ES%--0JN54IJJr$   zself: jt_all, size: anyc           
      6   t        | ||d      \  }}|j                  d      }|j                  d      }t        |      |j                         k  r?|j	                  g d t        |j                         t        |      z
        D        |      S t        d      )NTr   r   r   c              3       K   | ]  }d   yw)r   Nr1   )r   r   s     r   r   zbroadcast_to.<locals>.<genexpr>#  s     E1QEs   zwbroadcast_to(): broadcasting to a higher-dim shape is currently not supported for nested tensors with the jagged layout)r   r   r^   r!   r   r'  r_   )rd   re   rf   r   r   r   r   s          r   broadcast_tor    s    &4TMAz ..
!C>>&!D
4yCGGIzzMEcggi#d).C(DEMMNN
	4 r$   ztensors: anyc           
      T   t        | ||d      \  }}|j                  d      }t        |      dk(  rt        d      t        |      dk(  r|d   S g }t	        j
                  d |D         }t        d |D              }|D ]  }	|	j                  r!|j                  |	j                  |             0|	j                         t        |      k  rH|j                  t        |	j                  |j                  j                        fi t        |             t        d	       t        |      S )
NTr   r  r   z7broadcast_tensors(): expected at least one tensor inputr   c              3   4   K   | ]  }|j                     y wr   )r   r   rS   s     r   r   z$broadcast_tensors.<locals>.<genexpr>8  s     .H1qww.Hs   c              3   B   K   | ]  }t        |t              s|  y wr   rP   r  s     r   r   z$broadcast_tensors.<locals>.<genexpr>:  s     AQZ<%@qAr   zubroadcast_tensors(): broadcasting nested tensors with dense tensors of equal or higher dim is not currently supported)r   r   r^   r_   rF   broadcast_shapesr   r  r   r  r!   r	   r   r   r   r   )
rd   re   rf   r   r   r  outsbroadcast_shaper   rS   s
             r   broadcast_tensorsr  +  s   &4TMAz nnY'G
7|qRSS
7|qqzD,,.H.HIO
A'A
AC ;;KK78UUWs?++KKQ^^CKK,=,=>V.QTBUV ;  ;r$   z(condition: jt_all, self: any, other: anyc                 :   t        | ||d      \  }}|j                  d      }|j                  d      }|j                  d      }t        j                  |||      \  }}}t	         | |j
                  |j
                  |j
                  fi |fi t        |      S )NTr   	conditionr   r  )r   r   rF   r  r	   r   r   )rd   re   rf   r   r   r  r   r  s           r   
where_selfr  K  s     '4TMAz {+I
..
!CNN7#E "33IsEJIsEYU]]IjI

# r$   zself: jt, device: any?c                     t        | ||d      \  }}|j                  d      }t         | |j                  fi |fi t	        |      S r  r  r  s         r   _pin_memory_defaultr  `  r   r$   c                 n    t        | ||d      \  }}|j                  d      } | |j                  fi |S r  rY  r  s         r   is_pinned_defaultr  k  r~  r$   zself: jt_all, other: jt_allc                 @    |d   j                   |d   j                   k(  S r   )rx   r  s      r   is_same_size_defaultr  v  s     7==DGMM))r$   c                    t        | ||d      \  }}|j                  d      |d   d u xs, t        |d   t        t        f      xr t        |d         dk(  }|rt | j                  fi |}|j                  dd      rMt        |t        t        f      r t        |      fd|D              }|S |j                  j                        }|S |d   }	t        |d   t        t        f      }
|
s|	g}	t        j                         |	| j                        \  }}}}|
s|d   }||d<   |rj                  t        | d	      dd
lm} |r|r1 | j                  fi |}|j                  dd      r
 |d |      }|S |rt        | d      |j                  d       |
rj                  gnj                  } | j#                  |      fd|i|S |rt        | d       | j                  fi |}t%              |j                  dd      sKt        |d   t        t        f      r|d   n|d   g}|D ]"  }|j                  dz
  k  sdxx   dz  cc<   $  |fd|      S )NTr   r   r!   r   r%  Fc              3   T   K   | ]  }|j                  j                         ! y wr   )r  rN   )r   or   s     r   r   z#_apply_reduction.<locals>.<genexpr>  s     JCOO <Js   %(zf(): reducing across the ragged dimension is not supported for non-contiguous nested tensors with holes)tree_mapc                 $    | j                  d      S )Nr   )r  )r  s    r   rJ   z"_apply_reduction.<locals>.<lambda>  s    Q r$   zW(): reducing along a ragged and non-batch dimension is not supported for nested tensorszg(): reducing along the batch dimension but not the ragged dimension is not supported for nested tensorsr   rN   c                     t        | fi S r   r   )r  
out_kwargss    r   rJ   z"_apply_reduction.<locals>.<lambda>  s    ,q"?J"? r$   )r   r   r   r   r   r^   r   r   r   r  rN   r;   r!   rM   r&   torch.utils._pytreer  r  r   )rd   	func_nameidentity_elementre   rf   r   r   full_reductionoutdim_to_convert
is_dimlistconverted_dimrg  rh  reduce_on_non_batchr  dim_to_passdimlistr   r   r  s                      @@r   rz  rz  }  s   &4TMAz ..
!C  &$. :e$udm4TZ=N9OST9T  3;;-*->>)U+#t}-d3iJcJJ 
 mmCOO4
  &NJu-t}=J() 		+		 %a(%JuCLL4k ; ;
 	

 - s{{1j1C~~i/7=J #" k ": :  NN5!/93??+sK$$%56<GKU  + K K  3;;-*-#C(
~~i/ j/%? 5! '( 
  3s**}-2-3 ?EEr$   c                 n    t        | ||d      \  }}|j                  d      } | |j                  fi |S r  rY  r  s         r   sum_defaultr    r~  r$   z3self: jt_all, dim: any?, keepdim: any?, dtype: any?c                 $    t        | ddg|i |S )Nr   r   ry  r  s      r   sum_dim_IntListr    s    
 D%<T<V<<r$   z"self: jt_all, dim0: any, dim1: anyc           
      6   t        | ||d      \  }}ddlm} |j                  d      } ||j	                         |d   |d   f      \  }}||j
                  k(  s||j
                  k(  r|dk(  s|dk(  rt        d      ||j
                  k(  r|}	n|}	t        |      }
|	|
d	<   t        |j                         j                  t        t        |j                        ||j
                        t        t        |j                        ||j
                              fi |
S t        |j	                         |d   |j
                  d
      |d<   t        |j	                         |d   |j
                  d
      |d<   t         | |j                  fi |fi t        |      S )NTr   r   r   r   dim0dim1z?Transpose is not supported on the batch dimension for jagged NTrN   	transpose)r   r   r   r   r!   rN   r_   r   r	   r  r   r   r^   rx   r.   r   )rd   re   rf   r   r   r   r   r  r  to_dimrH  s              r   transpose_intr    s    '4TMAz 6
..
!C"3779z&/A:fCU.VWJD$
 s$#//"919	Q  3??"FF#C(
$*
=!JJL""#C		ND#//J#C		ND#//J

 
 	
 *	:f%sJv *	:f%sJv S[[7J7O>#;NOOr$   zself: jt_all, dims: anyc                    t        | ||d      \  }}|j                  d      }|j                  d      }t        |      }t        |j                        }|t        |      k7  rt        d| ddt        |       dz         d	d
lm}	  |	||      }
t        |
      t        t        |
            k7  rt        d      |j                  t        d      |
d	   d	k7  rt        d      |
j                  |j                        |d<   |
dd  D cg c]  }t        |||j                         }}||d<   t         | |j                  fi |fi |S c c}w )NTr   r   r5   z5permute(): number of dimensions in the tensor input (z) zAdoes not match the length of the desired ordering of dimensions ().r   r   z*permute(): duplicate dims are not allowed.zBpermute(): not supported on jagged layout nested tensor with holesz=Permute is not supported on the batch dimension for jagged NTrN   r   )r   r   r   r^   rx   r_   r   r   setrM   indexrN   r   r	   r   )rd   re   rf   r   r   r   r5   rH  inp_dimr   canonicalized_dimsr!   
inner_dimss                r   permute_defaultr
  '  s}   &4TMAz ..
!C>>&!D$J#))nG #d)CG9BOQRUVZR[Q\\^_`
 	

 6*7D9
#c*<&=">>EFF
||P
 	
 !!K
 	
 !3 8 8 IJ} &ab) 	GS#//:J  $JvS[[7J7F:FFs   Ec                   	 t        | ||d      \  }}|j                  d      |j                  d      	j                  dk7  r=t        j                        t        	      k7  rt        dj                   d	 d      t        	      d	k  st        	      st        d
j                   d	       	fd}t        t        	      dz
        D cg c]
  } ||       }}t        j                  j                               5  t         | j                  |      fi t              cd d d        S c c}w # 1 sw Y   y xY w)NTr   r   r   r   zVview(): does not support ragged_idx != 1 except when inp._size == size. inp._size is (z) and size is (r  r  zview(): cannot view shape z as c                 n    | j                   dz
  k(  rj                  j                  |       S | dz      S rL   )rN   r   r   )	inner_idxr   r   s    r   get_inner_sizez$view_default.<locals>.get_inner_sizet  s8    !++;;##I..	A&&r$   )r   r   rN   r   rx   r&   r^   r   r'  rF   inference_modeis_inferencer	   r   r   )
rd   re   rf   r   r   r  rk   
inner_sizer   r   s
           @@r   view_defaultr  P  s>   
 '4TMAz ..
!C>>&!D
!cii 0E$K ? YYKtfB@
 	
 4y1}.sD97		{$tfMNN ' .33t9q=-AB.#BJB 
		c..0	1 RDj9Q^C=PQR R	 CR Rs   D;&E  E	zHinput: jt_all, normalized_shape: any, weight: any?, bias: any?, eps: anyc                 P   t        | ||d      \  }}|j                  d      }|j                         dk  rt        d      |d   }|j                  |j
                     }|j                         t        |      z
  }|dk(  rt        d      ||v r|j                  t        d	      ||v rMt        j                  j                  j                  |j                  j                  |j
                  
      |j                  g|j                  g      }	t        j                  j                  j                  t        j                   |j                  j                  d   df|j"                  |j$                        |j                  g|j                  g      j'                  |	j                        }
|j                  j)                         j+                  d      j+                  d      |	j                  d   z  }t        j,                  |	dd      |z  }|	|z
  |
z  }t        j,                  t        j.                  |      dd      |z  }t        j0                  ||d   z         }||z  }t        j                  j                  j3                  ||j                  g|j                  j                  d         j5                  d|j                  |j
                  dz   d        }t7        |fi t9        |      ||fS  | |j                  fi |\  }}}t7        |fi t9        |      ||fS )NTr   r   r   zOlayer_norm(): not supported for NestedTensor objects with 2 or fewer dimensionsr   r   zVlayer_norm(): not supported when normalizing over the batch dimension for NestedTensorzklayer_norm(): not supported where lengths is not None if operating on the ragged dimension for NestedTensorr   )r]  r   )r7  dtype)r   r   )r!   r%  r   r`  rA   )r   r   r!   r&   r   rN   r^   rM   rF   r   r   rc  r   r   r   rd  onesr7  r  r   r  r  r   squaresqrtrf  	unflattenr	   r   )rd   re   rf   r   r   r   r   ragged_sizenum_dims_not_normalizedpadded_inputpadded_maskragged_lengthsmeanpadded_normalizedvariancestdpadded_layer_normjagged_layer_norm_valuesr#   s                      r   native_layer_norm_defaultr%    s"   
 '4TMAz ..
!C
wwyA~]
 	
 ""45))COO,K!ggi#.>*?? 	 1$d
 	
 &&3<<+Cy
 	

 	''yy~~EEKK//    \\N) F 
 iinnDDJJ))!,a0399U\\N) E 
 &
	 	 LL))!,66q9L<N<Nq<QQ 	
 II
  	 4
 II./
  	 jjJu$556-3#(99>>#Q#Q\\NKK%% $R $
 )		#//A-/0
 	! 1I^C5HI
 	
 S[[7J7FD#7>##67sCCr$   zpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc                     t        | ||d      \  }}|j                  d      }|j                  d      } | |j                  |j                  fi |\  }}}	|d ||	fS t        |fi t	        |      ||	fS )NTr   rn  r   rq  )
rd   re   rf   r   r   rn  r   d_inputd_gammad_betas
             r   "native_layer_norm_backward_defaultr*    s    
 '4TMAz ~~j)H
..
!C#H$4$4ckkPZPGWfgv&&8N3$78'6JJr$   z"self: jt_all, dim: any, index: anyc                    t        | ||d      \  }}|j                  d      }t        |j                         |d   |j                  dd      \  |d<   }|r|j                         |d      S |j                  t        d      t        |      }|d   |j                  d	z
  k  r|d
xx   d	z  cc<   t         | |j                  fi |fi |S )NTr   r   r!   selectr  r  zSselect(): not yet supported on dim != 0 for non-contiguous nested tensor with holesr   rN   )r   r   r.   r!   rN   r  rM   r_   r   r	   r   )rd   re   rf   r   r   r   r-   r  s           r   
select_intr-    s    &4TMAz ..
!C,<	:e$cooxQU-)Ju) zz|Jw/00
||a
 	

  $J%3??Q..=!Q&!S[[7J7F:FFr$   z7self: jt, dim: any?, start: any?, end: any?, step: any?c                     t        | ||d      \  }}|j                  d      }t        |j                         |d   |j                  d      |d<   t         | |j                  fi |fi t        |      S )NTr   r   r!   slice)r   r   r.   r!   rN   r	   r   r   r  s         r   slice_tensorr0    sw    
 '4TMAz ..
!C(	:e$coowJu S[[7J7O>#;NOOr$   z8input: jt_all, indices: any, values: t, accumulate: any?c                    t        | ||d      \  }}|j                  d      }|j                  d      }t        |      |j                         k  sJ t        |      |j                  dz   k  r:|j                         st        d      ddlm} |j                  }|j                  }	|	}
|j                  j                  |j                  dz
     }|
|}
g |j                  d |j                   |
|j                  |j                  dz   d  }|j                  d|	      } | | ||fi ||j                  |j                  |||	
      }| t        j                   j"                  j$                  j&                  k(  r+|j                  j)                  |j+                                |S |S |j-                         |j/                         j1                         }n|j-                         }t        j2                  t        j4                  ||j                     |k        d       |j/                         |d      ||j                     z   }|d|j                   |gz   ||j                  dz   d  z   }| t        j                   j"                  j$                  j&                  k(  r | |j                  |fi ||_        |S t7         | |j                  |fi |fi t9        |      S )NTr   r   indicesr   z[index_put(): If ragged dimension is not part of indices, this only works on contiguous NJTsr   r  r  r   z7Some indices in the ragged dimension are out of bounds!r   )r   r   r^   r!   rN   is_contiguousr&   r   r   r   r   r   r   r  r   rF   r   r   
index_put_r   rK  r  r   r   r  _assert_asyncr~   r	   r   )rd   re   rf   r   r   r   r2  r   r   r   r   r   r  
padded_inpnew_njtr   ragged_indicesfunc_indicess                     r   r4  r4  '  s    '4TMAz #w/C nnY'Gw<3779$$$
7|coo))  "m  	6**
**
!++##COOa$78"L
YY()

 YYs*,-

 ))#<)H
$W3
3LL!!
 599>>,,444KKgnn./J
 {{}++-$$&++-			'#//*W45A [[]71:.1IIN 	COO$
	 #//A%'
(		)  uyy~~((0003;;C
C
S[[,5*5

 r$   z{input: jt, weight: t, bias: t?, stride: any, padding: any, dilation: any, transposed: any, output_padding: any, groups: anyc                     t        | ||d      \  }}|j                  d      }t         | |j                  fi |fi t	        |      S r  r  r  s         r   convolution_defaultr;    sN     '4TMAz ..
!CS[[7J7O>#;NOOr$   c                 T   t        | ||d      \  }}|d   }t        |j                         |d   d|j                        \  }}}}|r|s|rJ |d   }	d|d<   t	        t
        j                  j                  j                  j                  ddfi |}
|j                  |j                  n|j                  j                         }t        |
j                         dz
        D ]  }|j                  d	      } |
|z  }|	s|j                  |j                        }|S d
}t	        | d|fi |S )NTr   r   r!   r  r%  r   r   rA   gzG?)r   r;   r!   rN   rz  rF   r   r   r   dim_IntListrM   r   r  r'  r  r   )rd   re   rf   r   r   r   rg  rh  r  r%  intermediate_sumr   r  intermediate_values                 r   mean_dimr@    sA    '4TMAz W
CBS	5	C?Q)+> &&&Y' $
9+IINN**FA
9C

 #&,,":#,,@Q@Q@S'++-12 	,A''+G	,(++coo.C

 D&*<K
KKr$   c                 n    t        | ||d      \  }}|j                  d      } | |j                  fi |S r  rY  r  s         r   mean_defaultrB    r~  r$   z&self: jt_all, dim: any?, keepdim: any?c                 $    t        | ddg|i |S )Nr4   Fry  r  s      r   any_dimsrD    s    D%@@@@r$   z%self: jt_all, dim: any, keepdim: any?c                     t        | ||d      \  }}|d   g|d<   t        t        j                  j                  j
                  j                  fi |S NTr   r!   )r   rD  rF   r   r   r4   r5   rd   re   rf   r   r   s        r   any_dimrH    Q    &4TMAz
 $E*+JuEIINN&&++:z::r$   c                 $    t        | ddg|i |S )Nr~   Try  r  s      r   all_dimsrK    s    D%????r$   c                     t        | ||d      \  }}|d   g|d<   t        t        j                  j                  j
                  j                  fi |S rF  )r   rK  rF   r   r   r~   r5   rG  s        r   all_dimrM    rI  r$   c                 n    t        | ||d      \  }}|j                  d      } | |j                  fi |S r  rY  r  s         r   all_any_max_min_defaultrO    s?     '4TMAz ..
!C*z**r$   c                     t        | ||d      \  }}t        j                  |d   j                        j                  }t        | d|g|i |S )NTr   r   minr   rF   finfor  maxrz  rd   re   rf   r   r   	dtype_maxs         r   min_dimrW    S    &4TMAz Jw/556::ID%DTDVDDr$   c                     t        | ||d      \  }}t        j                  |d   j                        j                  }t        | d|g|i |S )NTr   r   rT  r   rF   rS  r  rQ  rz  rd   re   rf   r   r   	dtype_mins         r   max_dimr]    rX  r$   c                     t        | ||d      \  }}t        j                  |d   j                        j                  }t        | d|g|i |S )NTr   r   aminrR  rU  s         r   amin_defaultr`    U     '4TMAz Jw/556::ID&)EdEfEEr$   c                     t        | ||d      \  }}t        j                  |d   j                        j                  }t        | d|g|i |S )NTr   r   amaxrZ  r[  s         r   amax_defaultrd    ra  r$   c                     t        | ||d      \  }}t        j                  |d   j                        j                  }t        | d|g|i |S )NTr   r   argminrR  rU  s         r   argmin_defaultrg  %  U     '4TMAz Jw/556::ID(IGGGGr$   c                     t        | ||d      \  }}t        j                  |d   j                        j                  }t        | d|g|i |S )NTr   r   argmaxrZ  r[  s         r   argmax_defaultrk  1  rh  r$   zAgrad: jt_all, dim: any, indices: jt_all, sizes: any, keepdim: anyc                   
 ddl m
 t        | ||d      \  }}|j                  d      }|j                  |d<   |j                  d      }|j                  |d<   t        
fdt        |d         D              }t        t        |d         |d	   |d
      |d	<   |j                  d      }|j                  j                  |j                  dz
        ||<   |dd  }||d<   t        |      }	||	d<   t         | di |fi |	S )Nr   r   Tr   gradr2  c              3   :   K   | ]  \  }} |      s|  y wr   r1   )r   rk   r}   r   s      r   r   z=value_selecting_reduction_backward_default.<locals>.<genexpr>M  s     WDAqmTUFVaWs   sizesr!   "value_selecting_reduction_backwardr   rN   r1   )r   r   r   r   r   r   ra   r.   r^   r   rN   r   r	   )rd   re   rf   r   r   rm  r2  r2   ro  rV  r   s             @r   *value_selecting_reduction_backward_defaultrq  =  s   
 D&4TMAz >>&!DJvnnY'G#OOJyWIj.A$BWWJ(Jw 5,	Ju NN7#E,,W-@-@1-DEE*!"IEJw"7+M#-M- *z*<m<<r$   c                    t        | ||d      \  }}|j                  d      }|D ]p  }t        |t              st	        d      |j                         |d   j                         k7  rt	        d      t        ||d   j                        rgt	        d       t        |d   j                         dz   |d	   |d   j                  d
      |d	<   t         | |D cg c]  }|j                   c}fi |fi t        |d         S c c}w )NTr   r  z+stack(): expected all nested tensors inputsr   z9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurer   r!   r  )r   r   r   r	   r&   r!   r   r   r.   rN   r   r   )rd   re   rf   r   r   r  rS   s          r   stack_defaultrs  a  s   &4TMAz
 nnY'G !\*LMM557gajnn&&K  "!WQZ%5%56X  )
1j/1G1GJu )Aaii)8Z8<J7ST:<V )s   DzQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?c                     t        | ||d      \  }}|j                  d      }|j                  d      }t         | ||j                  fi |fi t	        |      S )NTr   r2  r   r  )rd   re   rf   r   r   r2  r   s          r   embedding_defaultru    sf    
 '4TMAz
 nnY'G^^H%FVW__3
37Eg7N r$   zYgrad_output: jt, indices: jt, num_weights: any, padding_idx: any, scale_grad_by_freq: anyc                     t        | ||d      \  }}|j                  d      }|j                  d      } | |j                  |j                  fi |S )NTr   r2  r"  rY  )rd   re   rf   r   r   r2  r"  s          r    embedding_dense_backward_defaultrw    sU    
 '4TMAz nnY'G../K##W__C
CCr$   c                 |    t        | ||d      \  }}|j                  d      }|j                  j                         S r  )r   r   r   detachr  s         r   values_defaultrz    s@     '4TMAz ..
!C ;;r$   c                 l    t        | ||d      \  }}|j                  d      } | |j                        S r  rY  r  s         r   all_defaultr|    s8    &4TMAz ..
!Cr$   z-self: jt_all, padding: any, output_size: any?c                 t   t        | ||d      \  }}|j                  d      }|j                  t        d      |d   }|||j                     }n3|j
                  |j                  n|j                  j                  d      }|j                         }|j                  dkD  r|j                  |j                  dz
  d      }|j                  }	|j                         dkD  r|j                  d	      }n$|j                         dk(  r|j                  d
      }|j                  t         j"                  u }
|
r+|j$                  r|j'                  t         j(                        }t         j*                  j,                  j/                  ||j0                  g|g|d         }|
r+|j$                  r|j'                  t         j"                        }t3        |	      dkD  r|j5                  d
|	dd        }nt3        |	      dk(  r|j7                  d
      }|j                  dkD  r|j                  |j                  d      }|S )NTr   r   z?to_padded_tensor(): not supported for nested tensors with holesr  r   r   r   r  rA   padding)r   r   rM   r&   rN   _max_seqlen_tensorrd  r   r   r  r   r   r!   r   r  r  rF   boolis_cudar=  halfr   r   rc  r   r^   r  r   )rd   re   rf   r   r   r   r  max_seq_lenr  values_shapeis_bool
padded_outs               r   to_padded_tensor_defaultr    s   
 '4TMAz ..
!C
||M
 	

 ]+K!#//2 %%1 OO!!!$ 	 ZZ\F
!!#//A"5q9<<Lzz|a!,		!!"% llejj(G6>>5::&??		9	J :%%]]5::.
 <1))"l12.>?
	\	a	''+

))#//1=
r$   zcpadded: t, offsets: t, dummy: jt, ragged_idx: any?, min_seqlen: any?, max_seqlen: any?, sum_S: any?c                    t        | ||d      \  }}|d   |d   }}|j                  dd      }|dkD  r|j                  |d      }|j                  }|j	                         dkD  r|j                  d	      }n$|j	                         dk  r|j                  d
      }|j                  t        j                  u }	|	r+|j                  r|j                  t        j                        }t        j                  j                  j                  ||g|d         }
|	r+|
j                  r|
j                  t        j                        }
t!        |      dkD  r|
j#                  d
|dd        }
nt!        |      dk  r|
j%                  d
      }
|dkD  r|
j                  |dz
  d      }
|d   }|d   }i }|||d<   |||d<   t'        |
|||      S )NTr   paddedr   r2   r   r  r   r  rA   r   r   r   r   )rN   r   )r   r   r   r   r!   r   r  r  rF   r  r  r=  r  r   r   rf  r^   r  r   r	   )rd   re   rf   r   r   r  r   r2   padded_ragged_dim1_shaper  r  r   r   metadata_caches                 r   "_nested_from_padded_tensor_defaultr    s   
 '4TMAz !*Jy,AGFa0J A~!!*a0%||zz|a!,		!!"% llejj(G6>>5::&YY^^;;	:g.F 6>>5::& #$q(!!"&>qr&BC	%	&	*#A~!!*q.!4L)JL)JN'1|$'1|$&	 r$   zcvalues: t, offsets: t, dummy: jt_all, lengths: t?, ragged_idx: any?, min_seqlen: t?, max_seqlen: t?c                     t        | ||d      \  }}|d   |d   |d   }}}|d   }|d   }	|d   }
i }|	|	|d<   |
|
|d<   t        |||||	      S )
NTr   r   r   r   r2   r   r   )r   rN   r   )r   r	   )rd   re   rf   r   r   r  r   r   r2   r   r   r  s               r    _nested_view_from_jagged_defaultr  6	  s    
 '4TMAz
 	799 GF
 L)JL)JL)JN'1|$'1|$& r$   c                 `    t        | ||d      \  }}|j                  d      }|j                  S r  )r   r   r   r  s         r   _nested_get_offsetsr  V	  3    &4TMAz ..
!C<<r$   c                 `    t        | ||d      \  }}|j                  d      }|j                  S r  )r   r   rM   r  s         r   _nested_get_lengthsr  `	  r  r$   c                 `    t        | ||d      \  }}|j                  d      }|j                  S r  )r   r   rN   r  s         r   _nested_get_ragged_idxr  j	  s3    &4TMAz ..
!C??r$   c                     t        | ||d      \  }}|j                  d      }|j                  j                  dd       S )NTr   r   r   r   r   r   r   r  s         r   _nested_get_min_seqlenr  t	  B    &4TMAz ..
!C""<66r$   c                     t        | ||d      \  }}|j                  d      }|j                  j                  dd       S )NTr   r   r   r  r  s         r   _nested_get_max_seqlenr  ~	  r  r$   zself: jt, mask: anyc                 (   t        | ||d      \  }}|j                  d      }|j                  d      }|j                  dkD  rt        d      |j                  |j                  k7  r%t        d|j                   d|j                         |j
                  j                  |j                               }t        j                  |j                         j                  d	
      d      }t        |      }||j                     |d<   t        dd|i|S )NTr   r   rv  r   z3masked_select only support 2-D selections currentlyzMask with shape z& is not compatible with input's shape r   r_  r  r   r  r1   )r   r   r   r&   r   r   masked_selectr  r  r  r  r   r   r	   )	rd   re   rf   r   r   r   rv  
res_valuesmask_cumsums	            r   masked_select_defaultr  	  s   &4TMAz ..
!C>>&!D
xx!|PQQ	djj	 tzzl*PQTQZQZP[\
 	
 **4;;=9J%%,,,3V<K#D!#,,/DO 
 r$   z2grad_output: t, self: jt_all, dim: any, index: anyc                     t        | ||d      \  }}|j                  d      }|j                  d      }t        j                  ||j                        }|j                  |d   |d         j                  |       |S )NTr   r   r"  )r  r!   r  )r   r   rF   
zeros_liker  r,  rK  )rd   re   rf   r   r   r   r"  
grad_inputs           r   _nested_select_backward_defaultr  	  s|    
 '4TMAz ..
!C../K!!#[->->?Jj'G)<=CCKPr$   zself: jt_all, s: anyc                     |d   }|d   } | |j                   |        | |j                  |       |j                   | |j                  |       y y r   )r   r   rM   )rd   re   rf   r   streams        r   record_stream_defaultr  	  sM    
q'C!WFfv
||S\\6"  r$   zRself: jt_all, size: any, dtype: any?, layout: any?, device: any?, pin_memory: any?c                     t        | ||d      \  }}|j                  d      }t        |d         dk(  r | |j                  fi |S t	        d      )NTr   r   r   r   z2new_empty() not supported for NJT with shape != ())r   r   r^   r   r&   r  s         r   new_empty_defaultr  	  s]     '4TMAz ..
!C
:f!#CKK.:..
K
LLr$   zself: jt_all, ...c                 j    t        d |D              }t         | d |D        i |fi t        |      S )Nc              3   B   K   | ]  }t        |t              s|  y wr   rP   r   s     r   r   z&activation_backward.<locals>.<genexpr>	  s     Lsjl.KsLr   c              3   X   K   | ]"  }t        |t              r|j                  n| $ y wr   )r   r	   r   r   s     r   r   z&activation_backward.<locals>.<genexpr>	  s"     TZ\:ckkCTs   (*r   )rd   re   rf   r"  s       r   activation_backwardr  	  sH     LdLLKTtT	
	

 
% r$   zself: jt_all, value: anyc                     t        | ||d      \  }}|j                  d      }t         | |j                  fi |fi t	        |      S r  r  r  s         r   fill_Scalarr  	  r   r$   c                 r    t        | ||d      \  }}|j                  d      } | |j                  fi | |S r  rY  r  s         r   fill__Scalarr  	  s?    &4TMAz ..
!C#
#Jr$   c                     t        | ||d      \  }}|j                  d      }t        |      } | |j                        \  }}t	        |fi |t	        |fi |fS r  )r   r   r   r   r	   )	rd   re   rf   r   r   r   rV  mantissaexponents	            r   frexp_Tensorr  
  so    &4TMAz ..
!C"3'Mckk*Hh2M2L5!5  r$   z+grad: any, self: any, other: any, mask: anyc                 p   t        | ||d      \  }}|j                  d      }|j                  d      }|j                  d      }|j                  d      }|yd }	|d   r&t        j                  ||j	                  d	d
            }	d }
|d   r&t        j                  |j	                  d	d
      |      }
|	|
fS )NTr   rm  r   r  rv  r   r   rA   r   )r   r   rF   r&  r   )rd   re   rf   r   r   rm  r   r  grad_input_mask	grad_self
grad_others              r   matmul_backward_defaultr  
  s    
 '4TMAz >>&!D
..
!CNN7#E nnV,O|IqLLur2'>?	Jq\\#--B"7>
z""r$   )flex_attentionflex_attention_backward)GraphModulequerykeyr  	score_mod
block_maskscalekernel_optionsscore_mod_other_buffersmask_mod_other_buffersc	                    | j                         dk(  r&|j                         dk(  r|j                         dk(  sJ t        d ||z   D              rt        d      t        | j	                         j                  d      |j	                         j                  d      |j	                         j                  d      ||||||	      }	t        j                  j                  |	d   j                  dd      j                  d      | j                  | j                  | j                  | j                        j                  dd      }
t        j                  j                  |	d   j                  dd      j                  d      | j                  | j                  | j                  | j                        j                  dd      }|
|fS )	N   c              3   l   K   | ],  }t        |t        j                        xr |j                   . y wr   )r   rF   rG   r  )r   bufs     r   r   zflex_njt.<locals>.<genexpr>C
  s/       	3%7#--7s   24zflex_attention(): Nested tensor score_mod / mask_mod buffers are not currently supported. Please file an issue if this is important to you.r   )r  r  r  r  r  r  r   r   r   r   )r!   r4   r&   flex_attention_hopr  r  rF   r  nested_tensor_from_jaggedr   r   r   rM   r   r   )r  r  r  r  r  r  r  r  r  r#   
output_njtlogsumexp_njts               r   flex_njtr  4
  s    99;!	Q599;!3CCC  *-CC  U
 	
    #

q!  #% 75
F 77q	Aq!))!,**** 8  i1o  LL::q	Aq!))!,**** ;  i1o  &&r$   r  	logsumexprn  grad_logsumexpfw_graphjoint_graph.c                    t        | j                         j                  d      |j                         j                  d      |j                         j                  d      |j                         j                  d      |j                         j                  d      |j                         j                  d      |j                         j                  d      |||	|
|||      }|\  }}}}t        j                  j                  |j                  dd      j                  d      | j                  | j                  | j                  | j                        j                  dd      }t        j                  j                  |j                  dd      j                  d      |j                  |j                  |j                  |j                        j                  dd      }t        j                  j                  |j                  dd      j                  d      |j                  |j                  |j                  |j                        j                  dd      }||||fS )Nr   )r  r  rn  r  r  r  r  r  r  r  r  r   r   r  )flex_attention_backward_hopr  r  rF   r  r  r   r   r   rM   r   r   )r  r  r  r  r  rn  r  r  r  r  r  r  r  r  r#   dense_q_graddense_k_graddense_v_gradscore_mod_other_buffer_grads
njt_q_grad
njt_k_grad
njt_v_grads                         r   flex_njt_backwardr  m
  s   & )  #

q!  #JJL""1%""$..q1",,Q/%,,.88;% 75F$ NTJL,.J77q!$,,Q/**** 8  i1o  77q!$,,Q/(((( 8  i1o  77q!$,,Q/**** 8  i1o  
J0LMMr$   z	self: anyc                     ddl m}  |       S )Nr   )_nt_view_dummy)$torch.nested._internal.nested_tensorr  )rd   re   rf   r  s       r   _nested_get_jagged_dummyr  
  s    Cr$   r   IMPLr  CPUCUDAMeta)F)TFF)r   )r1   r1   )r   r  r   typingr   rF   torch.nn.functionalra  rb  r  torch.fx.operator_schemasr   torch.nested._internal.sdpar   r   r	   r
   r   Any__annotations__r   Dictr   r.   r;   strrq   ry   r   r   r   r   register_jagged_funcCallabler   r   r   r   r   r   r   r   r   r   r!   r   r   r  r  r	  primr5  r  r   r  r3  r  r  r(  r  linearr  linear_backwardr/  r=  r  r1  _to_copyrI  rK  rS  ry  
empty_like	ones_liker  	rand_like
randn_likerW  	full_likerandint_like	low_dtypezero_rZ  _softmaxrl  _softmax_backward_dataro  native_dropoutrt  native_dropout_backwardrw  r  dim_intr{  r}  r[   rG   r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r&  r  bmmr  r   r  r  r  r  r  wherer$  r  _pin_memoryr  	is_pinnedr  is_same_sizer  rz  r   r  r=  r  r   r  permuter
  view_unsafe_viewr  native_layer_normr%  native_layer_norm_backwardr*  r,  r-  r/  r0  	index_putr4  convolutionr;  r  r@  rB  r4   r5   rD  rH  r~   rK  rM  rT  rQ  rO  rW  r]  r_  r`  rc  rd  rf  rg  rj  rk  rp  rq  r  rs  	embeddingru  embedding_dense_backwardrw  r  _nested_get_valuesrz  r|  r  r  _nested_from_padded_tensorr  _nested_view_from_jaggedr  r  r  r  r  r  r  r  _nested_select_backwardr  record_streamr  	new_empty	new_zerosnew_onesr  elu_backwardhardshrink_backwardhardsigmoid_backwardhardtanh_backwardsoftplus_backwardsoftshrink_backwardr  fillScalarr  fill_r  frexpr  matmul_backwardr  &torch._higher_order_ops.flex_attentionr  r  r  r  torch.fx.graph_moduler  py_implTuplere  r  Unionr  r  library_scoped_libraryimplr1   r$   r   <module>r+     s:           8 K ' c #% $sCx. %3. :Y8=S =D =@

#&
+7
AD
	
0. )y((8HI &HX,> &RKS\E$P 		33;;		''		""		$$		  ((		!!))		))11 00, eiinn++33^D E 
YY^^  !
	
 eiinn22::NKF LF( 	IINN  ..0S
 	IINN  "FEE2 eiinn++335UVP WP 	IINN""**<"	"J eiinn''--/JKP LP eiinn--55~F% G%P 	IINN  "Q: D UYY^^**22N C
 		!!))		  ((		!!))		  ((		!!)) 	,5	,5^ X UYY^^--557V W U UYY^^0088:S T 	IINN))+N
 eiinn**22NC D 	IINN##%QEPEPP 	IINN))11=	 	IINN!!))+N

 	IINN**22+			 	IINN8>	> eiinn))113NO+ P+ 	IINN!G" 	IINN##++-T" 	IINN!!#P77  eiinn**224VW&
 X&
R eiinn++//1JKE LEP eiinn,,002FGK HK eiinn..668PQE RE, eiinn((002JK L0 eiinn++335LMj NjZ eiinn((002KLE ME  	IINN!!#LNN  eiinn..668LMK NK eiinn1199;TU V" eiinn66>>O P> 	IINNI$ eiinn0088:RSP TP eiinn..668PQ+ R+ 	IINN'')F**gFT eiinn((002MN+ O+ 	IINN""9=	= 	IINN  "F'P'PT eiinn,,446OP%G Q%GP 
YY^^  %)).."="="E"EF,R	,R^ 	IINN$$,,N^D	^DB 	IINN--55v
K	
K eiinn++//1UVG WG8 	IINN=
P	
P 	IINN$$> 	IINN%%>Q		Qh 	IINN&&G
P
P 	IINNR#L#LL eiinn))113NO+ P+ eiinn((--/WXA YA eiinn((,,.UV; W; eiinn((--/WX@ Y@ eiinn((,,.UV; W; 		""		""		""		""	 ++ eiinn((,,.UVE WE eiinn((,,.UVE WE 	IINN!IFF 	IINN!IFF 	IINN!!#KHH 	IINN!!#KHH 	IINN55==G=	=@ eiinn**224LM N< 	IINN$$W	 	IINN++33_D	D 		%%		))11 	 	  eiinn((00.A B 	IINN##++38	8v 	IINN--55i2	2j 	IINN++33i	8 eiinn88@@.Q R eiinn88@@.Q R eiinn;;CC^T U eiinn;;CC^T7 U7 eiinn;;CC^T7 U7 eiinn22::<QR S0 	IINN**228	 eiinn22::<RS# T# 		  ((		  ((		''
 Y
M
M 		##++		**22		++33		((00		((00		**22 
	
	 eiinn))002LMP NP eiinn**113MN O eiinn**11>B C 	IINN""**1#	#0 . L) &($&5'<<5'	5' <<5' 	5'
 5' 5' cN5' #5' "5' 5<<%&5' *5'p %$$\2 &($&;N<<;N	;N <<;N 
	;N
 ||;N ll;N LL;N Hk)*;N ;N ;N ;N cN;N #;N ";N 	LL%,,eHU\\4JC4O.PP;N 3;N~ eiinn==EE{S T ]]""662 LdII(*BEJII(*BFKII(*BFKL L Ls   d&=Ae,e,Ae5