
    Vh                        U d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZm	Z	m
Z
mZmZmZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lm Z  er&d dl!Z!d dl"Z"d dl#m$Z% d dl&Zd dl'Zd dl(Zd dl)Zd dl*Zd dl+Zda,ee-   e.d<   g dZ/de-fdZ0i Z1e2e3e4f   e.d<   	 dPde5deejl                  jn                     de3fdZ8dee3   fdZ9de3fdZ:d Z;de3fdZ< ejz                  e>      Z? G d de      Z@dejl                  jn                  ddfdZAdeBe3df   fdZCdejl                  jn                  deBedf   fdZDdejl                  jn                  defd ZEdejl                  jn                  deBe3df   fd!ZFd"eBe3df   deBe3df   fd#ZGd$eBeej                  ej                  e4ej                  eKej                  e-f   df   deBd%   fd&ZMd'eBej                  df   d(eBd%   deBej                  df   fd)ZNd*edej                  fd+ZOd,eej                  e4eKe-f   d-d.dej                  fd/ZPd*ej                  d0eej                  ej                  e4ej                  eKej                  e-f   deej                  e4eKe-f   fd1ZQd2d3d4eBe3df   d5eBej                  df   d6eBd%   d7eBe3df   d8eBej                  df   d9eBd%   d:e-d;eBd<   d=eBeej                  ej                  e4ej                  eKej                  e-f   df   deBeej                  e4eKe-f   df   fd>ZRd2d3d4eBe3df   d5eBej                  df   d6eBd%   d7eBe3df   d8eBej                  df   d9eBd%   d:e-d;eBd<   d=eBeej                  ej                  e4ej                  eKej                  e-f   df   deBeej                  e4eKe-f   df   fd?ZS G d@ dA      ZTe j                   G dB dC             ZVee3eBe3ee3ef   f   f   ZWee.dD<   	  e j                  dEF       edGH       G dI dJ                    ZX edGH       G dK dL             ZY edGH      ddMdejl                  jn                  dNeeeXee3ef   f      fdO       ZZy)Q    N)MappingSequence)AnyCallableFinalOptionalTYPE_CHECKINGUnion)	TypeAlias)
FakeTensor)compatibility)FakeTensorProp)OperatorSupport)CALLABLE_NODE_OPS)_pytree_pybind_state_SUPPORT_ONNXRT)is_onnxrt_backend_supportedtorch_compile_backendOrtExecutionProviderOrtBackendOptions
OrtBackendreturnc                     t         d	 t        j                  d       t        j                  d       t        j                  d       ddl} ddl} ddl} ddl} ddlm}m	}m
}m} da t         S t         S # t        $ r
 da Y t         S w xY w)	a!  Returns ``True`` if ONNX Runtime dependencies are installed and usable
    to support TorchDynamo backend integration; ``False`` otherwise.

    Example::

        # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ONNX)
        >>> import torch
        >>> if torch.onnx.is_onnxrt_backend_supported():
        ...     @torch.compile(backend="onnxrt")
        ...     def f(x):
        ...             return x * x
        ...     print(f(torch.randn(10)))
        ... else:
        ...     print("pip install onnx onnxscript onnxruntime")
        ...
    Nonnxruntimezonnxruntime.capi._pybind_state
onnxscriptr   )decomposition_tablefx_onnx_interpreterpasses
type_utilsTF)r   	importlibimport_module
torch.onnxtorch.onnx._internal%torch.onnx._internal._exporter_legacy torch.onnx._internal.diagnosticstorch.onnx._internal.fxr   r   r    r!   ImportError)torchr   r   r    r!   s        P/home/dcms/DCMS/lib/python3.12/site-packages/torch/onnx/_internal/onnxruntime.pyr   r   /   s    & 	$##M2##$DE ##L1'83  #O ?  	$#O	$s   AA1 1BB_dumped_onnx_modelmodel_stringgraph_modulec                    t         j                  j                  dd      }|syt        j                  |d      dz   }| | d}t	        |d      5 }|j                  |        ddd       |t        |<   |D| | d}t	        |d	d
      5 }|j                  t        |j                               ddd       |S |S # 1 sw Y   ZxY w# 1 sw Y   |S xY w)a  Stores the onnx model into a file.
    The name is "{ONNXRT_DUMP_PATH}{N}.onnx"
    where *N* is the number of files already stored with
    this prefix.
    If graph_module is not None, the graph is stored as a string with
    the same filename except the extension (.txt).
    ONNXRT_DUMP_PATHN    z.onnxwbz.txtwzutf-8)encoding)osenvirongetr,   openwritestrgraph)r-   r.   prefixnfilenameffilename_txts          r+   _dump_onnx_modelrC   c   s     ZZ^^.5Fvr*Q.A!E"H	h	 	!"v !D),g6 	-!GGC**+,	-O8O 
	-Os   B;
%C;CCc                      dgS )NCPUExecutionProvider rF       r+   _infer_default_epsrH   |   s     ###rG   namec                     t         j                  j                         r*t         j                  j                  j	                  |        yy)zIf PyTorch is installed with CUDA support, this starts NVTX range.

    Check torch.cuda.nvtx.range_push's document for more details.
    N)r*   cudais_availablenvtx
range_pushrI   s    r+   _nvtx_range_pushrP      s/    
 zz 

""4( !rG   c                      t         j                  j                         r)t         j                  j                  j	                          yy)zIf PyTorch is installed with CUDA support, this terminates NVTX range.

    Check torch.cuda.nvtx.range_pop's document for more details.
    N)r*   rK   rL   rM   	range_poprF   rG   r+   _nvtx_range_poprS      s-    
 zz 

!!# !rG   device_typec                     ddl m} | dk(  r|j                  j                         S | dk(  r|j                  j	                         S | dk(  r|j                  j                         S t        d| z         )Nr   r   rK   cpumaiazUnsupported device type: )onnxruntime.capir   	OrtDevicerK   rV   npu
ValueError)rT   ORTCs     r+   _get_ort_device_typer]      sg    6f~~""$$e~~!!##f~~!!##
0;>
??rG   c                        e Zd ZdZdee   deeef   f fdZde	ee
j                  j                  f   de
j                  j                  def fdZ xZS )	OrtOperatorSupporta0  Operator support for ONNXRuntime backend.

    It has two-level of support decision. One is via support_dict and the other one
    is via extra_support_dict. The logic of using support_dict is implemented in
    OrtOperatorSupport and extra_support_dict is used by OperatorSupport.is_node_supported.
    support_dictextra_support_dictc                 2    t         |   |       || _        y N)super__init___onnx_support_dict)selfr`   ra   	__class__s      r+   re   zOrtOperatorSupport.__init__   s    
 	+,".rG   
submodulesnoder   c                    |j                   t        vry|j                   dk(  rM|j                  | j                  v r5t        j                  d|j                  t        |j                               yt        | !  ||      r5t        j                  d|j                  t        |j                               yt        j                  d|j                  t        |j                               y)NFcall_functionz0support_dict supports node.target: %s (type: %s)Tz6extra_support_dict supports node.target: %s (type: %s)zLsupport_dict and extra_support_dict don't support node.target: %s (type: %s))
opr   targetrf   loggerinfotyperd   is_node_supportedwarning)rg   ri   rj   rh   s      r+   rr   z$OrtOperatorSupport.is_node_supported   s     77++77o%$++9P9P*PKKBT[[!
  7$Z6KKHT[[!
 ZKK	

 rG   )__name__
__module____qualname____doc__setr   dictr<   re   r   r*   nnModulefxNodeboolrr   __classcell__)rh   s   @r+   r_   r_      sc    /SX /4S> /!#uxx"67?Dxx}}	 rG   r_   c                     | j                   }g }d}|j                  D ]7  }|j                  dk(  r|j                  |       |&|j                  dk7  s6|}9 |y|D ]  }|j	                  |        y)z
    In torch.fx.Graph, placeholder is a special assignment node. If it's not
    executed in the beginning, it could overwrite values computed by upstream
    nodes.
    Nplaceholder)r=   nodesrm   appendprepend)r.   r=   placeholdersfirst_not_placeholderrj   r   s         r+   _move_placeholder_to_frontr      s     EL  )77m#% (TWW-E$(!	)
 $# 3%%k23rG   .c                      g }| D ]]  }t        |d      s|j                  }|j                  dk(  r|j                  d       =|j                  dk(  sM|j                  d       _ t	        |      S )zBReturn the first valid device (i.e., GPU or CPU) in argument list.devicerK   CUDAExecutionProviderrV   rE   )hasattrr   rq   r   tuple)argsepsargr   s       r+   _infer_ep_from_devicer      sg    
C 33!ZZF{{f$

23%

123 :rG   c                    g }| j                   j                  D ]f  }|j                  dk(  st        |d      r7d|j                  v r)t        |j                  d   t        j                        sJ |j                  |       h t        |      S )Nr   metaval)
r=   r   rm   r   r   
isinstancer*   Tensorr   r   )r.   r   rj   s      r+   _extract_graph_module_inputsr      s{    L""(( &77m#tV$$)));!$))E"2ELLAAA%	&
 rG   c                     | j                   j                  D ]"  }|j                  dk(  s|j                  d   c S  t	        d      )zHCollect "val" fields from outputs metadata in this torch.fx.GraphModule.outputr   z2No output node found in this torch.fx.GraphModule.)r=   r   rm   r   r[   )r.   rj   s     r+   _extract_graph_module_outputsr     sG    ""((  77h 99Q<	 
 I
JJrG   c                     t        j                  t        |             \  }}|D cg c]+  }t        |d      rd|j                  v r|j                  d   - }}t        | S c c}w )z[Return the all valid devices (i.e., GPU or CPU) among outputs of this torch.fx.GraphModule.r   r   )r   tree_flattenr   r   r   r   )r.   flattened_output_args_
output_argselected_output_argss        r+   _infer_ep_from_graph_moduler     ss    &33%l3 1 0 J'EZ__,D	 	  !"677s   0A r   c                 f    dt         dt        fd}t        |       }t        t	        ||d            S )z:Sort execution providers in eps based on pre-set priority.epr   c                     | dk(  ry| dk(  ryy)NrE      r   r3   r   rF   )r   s    r+   get_execution_provider_priorityz2_sort_eps.<locals>.get_execution_provider_priority"  s    ''(( rG   T)keyreverse)r<   intrx   r   sorted)r   r   
unique_epss      r+   	_sort_epsr     s6    	C 	C 	 SJ
(GQUVWWrG   valueszORTC.OrtDevice.c           	      P   ddl m dt        dt        fddt        t        j
                  t        j                  t        t        j                  t        t        j                  t        f   dt        ffdt        |       dkD  rt        fd| D              }|S  d	      fS )
Nr   r   	device_idr   c                     | xs dS )Nr   rF   )r   s    r+   _device_id_or_zeroz-_get_onnx_devices.<locals>._device_id_or_zero;  s    ~ArG   valuec           	      8   t        | t        j                        rc j                  t	        | j
                  j                        j                  j                          | j
                  j                              S t        | t        j                  t        t        j                  t        t        j                  t        f      r5 j                  t	        d      j                  j                         d      S t        dt!        t        |             z         )NrV   r   zUnsupported value type: )r   r*   r   rY   r]   r   rq   default_memoryindexSymIntr   SymFloatfloatSymBoolr~   r[   r<   )r   r\   r   s    r+   _map_tensor_or_sym_to_devicez7_get_onnx_devices.<locals>._map_tensor_or_sym_to_device>  s    
 eU\\*!4>>$U\\%6%67--/"5<<#5#56 
 ELL#u~~uemmTR
 "4>>$U+T^^-J-J-La  7#d5k:JJKKrG   c              3   .   K   | ]  } |        y wrc   rF   ).0r   r   s     r+   	<genexpr>z$_get_onnx_devices.<locals>.<genexpr>S  s     TE8?Ts   r3   )rX   r   r   r
   r*   r   r   r   r   r   r~   lenr   )r   ort_devicesr\   r   r   s     @@@r+   _get_onnx_devicesr   1  s     7c c LLL%,,U^^UEMMSWW
L 
	L( 6{QTVTT,Q/11rG   tensorsdevicesc                 p   ddl m} ddlm}  |j                         }|j                  t        |              g }g }g }| D ]^  }|j                  ||j                            |j                  |j                                |j                  |j                                ` |j                  | ||||       |S )Nr   r   )_TORCH_DTYPE_TO_NUMPY_DTYPE)rX   r   "torch.onnx._internal.fx.type_utilsr   OrtValueVectorreserver   r   dtypesizedata_ptrpush_back_batch)	r   r   r\   r   	ortvaluesdtypesshapes	data_ptrstensors	            r+   !_get_ortvalues_from_torch_tensorsr   Y  s     7N###%Ic'l#FFI ,1&,,?@fkkm$*+, gy&&'JrG   r   c                     | j                   rt        d      t        j                  | j	                         | j
                  | j                        }|S )Nz#sparse tensor is not yet supported.)r   r   )	is_sparser[   r*   emptyr   r   r   )r   outs     r+   _to_real_tensorr   n  s<    >??
++fkkm6<<
NCJrG   dynamo_value
value_infoonnx.ValueInfoProtoc                    t        | t        j                        rZt        |j                  j
                  j                  j                        dk(  r$| j                  dk(  rt        j                  |       S t        | t              r%t        j                  | t        j                        S t        | t              r%t        j                  | t        j                        S t        | t              r%t        j                  | t        j                        S t        | t        j                        sJ | j                         S )z9Helper function to wrap PyTorch variables as torch.Tensorr   )r3   )r   )r   r*   r   r   rq   tensor_typeshapedimsqueezer   r   int64r   float32r~   
contiguous)r   r   s     r+   _adjust_scalar_from_fx_to_onnxr   u  s     	<.
++11556!;$& }}\**	L#	&||L<<	L%	(||L>>	L$	'||L

;;,555&&((rG   
prim_valuec           	          t        | t        j                        sJ d       t        |t        j                  t        t        j
                  t        t        j                  t        f      r| j                         S | S )zFHelper function to wrap ORT-produced torch.Tensor as PyTorch variableszORT's output must be tensor.)
r   r*   r   r   r   r   r   r   r~   item)r   r   s     r+   _adjust_scalar_from_onnx_to_fxr     sX    $ fell+K-KK+	sENNE5==$G
 {{}MrG   sessonnxruntime.InferenceSessioninput_namesinputsinput_devicesoutput_namesoutputsoutput_devicespreallocate_outputinput_value_infosr   .normalized_prim_outputsc
                    dd l }
ddlm} t        d       t	        d t        ||      D              }t                t        d       t        ||      }|rt	        d |D              }t        ||      }n |j                         }t                t        d        |
j                         }|j                  dd	       | j                  ||||||       t                |r3t        d
       t	        d t        |	      D              }t                |S t        d
       |
j                  j                  j                  j                  |      }t	        d t        ||	      D              }t                |S )Nr   r   r   c              3   :   K   | ]  \  }}t        ||        y wrc   r   r   r   r   s      r+   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  #      C 	'sJ7   r   c              3   V   K   | ]!  }t        |t              rt        |      n| # y wrc   )r   r   r   )r   ts     r+   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  s(      
GH*Q
";OAB
s   ')run_with_ortvaluevector'disable_synchronize_execution_providers1zafter run_with_ortvaluevectorc              3   :   K   | ]  \  }}t        ||        y wrc   r   r   onnx_outputprim_outputs      r+   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  #      
([ +;D
r   c              3   :   K   | ]  \  }}t        ||        y wrc   r   r   s      r+   r   z8_run_onnx_session_with_ortvaluevector.<locals>.<genexpr>  r  r   )r   rX   r   rP   r   ziprS   r   r   
RunOptionsadd_run_config_entryr   training	ortmodule_utils_ortvalues_to_torch_tensor)r   r   r   r   r   r   r   r   r   r   r   r\   
ort_inputspth_outputsort_outputsrun_optionss                   r+   %_run_onnx_session_with_ortvaluevectorr    sx   " 6\" "6+<= F &'26=IJ
  
LS
 
 8^T)d))+./(+((*K$$%NPST  [*lK  89  
,/=T,U
 
 	 	89!**44;;VV
  
,/=T,U
 
 	rG   c
           	      V   dd l }
t        d t        ||      D              }t        ||      D ci c]=  \  }}||
j                  j	                  |j                         j                               ? }}}| j                  ||      }t        d t        ||	      D              }|S c c}}w )Nr   c              3   :   K   | ]  \  }}t        ||        y wrc   r   r   s      r+   r   z/_run_onnx_session_with_fetch.<locals>.<genexpr>  r   r   c              3   `   K   | ]&  \  }}t        t        j                  |      |       ( y wrc   )r   r*   
from_numpy)r   r   r  s      r+   r   z/_run_onnx_session_with_fetch.<locals>.<genexpr>  s4      
 E;	 	'U#	
s   ,.)r   r   r  OrtValueortvalue_from_numpyrV   numpyrun)r   r   r   r   r   r   r   r   r   r   r   rI   r   feedr  r  s                   r+   _run_onnx_session_with_fetchr    s    "  "6+<= F  V4D& 	k""66vzz|7I7I7KLLD  ((<.K 
 #&k3J"K K s   AB%c                       e Zd ZdZdddeedf   ded   deedf   d	ed   d
ed   ded   deeej                  df   ej                  f   fdZ	d Z
y)OrtExecutionInfoPerSessionzWInformation required to execute torch.fx.GraphModule using onnxruntime.InferenceSessionsessionr   r   .r   r   r   output_value_infosr   r   r   example_outputsc	                 t    || _         || _        || _        || _        || _        || _        || _        || _        y rc   r  r   r   r   r  r   r   r  )	rg   r  r   r   r   r  r   r   r  s	            r+   re   z#OrtExecutionInfoPerSession.__init__+  sM     6= -8BS-9CU :G:H  	rG   c                    ddl m}m} t        |      t        | j                        k7  ryt        || j                        D ]  \  }}t        |t        j                  t        t        f      s yt        |t        t        t        f      ro |t        |            }||j                  j                  j                  k7  r yt        |j                  j                  j                  j                         dk7  r y||j"                     }||j                  j                  j                  k7  r yt        |j                  |j                  j                  j                  j                         D ]Z  \  }}t        |t              r|j$                  |k(  s|j&                  r2t        |t        j(                        r|j&                  rY  y  y)Nr   )(_TORCH_DTYPE_TO_ONNX_TENSOR_ELEMENT_TYPE,from_python_type_to_onnx_tensor_element_typeFT)r   r#  r$  r   r   r  r   r*   r   r   r   r~   rq   r   	elem_typer   r   r   	dim_value	dim_paramr   )	rg   r   r#  r$  r   r   
onnx_dtyper   onnx_dims	            r+   is_supportedz'OrtExecutionInfoPerSession.is_supportedL  sd   	
 t9D2233"4)?)?@ 	!OCcELL%#=> #UD12I$s)T
!<!<!F!FF z2288<<=B  B#))LJZ__88BBB!$SYY
0K0K0Q0Q0U0U!V !Xc3'&&#-1C1CU\\2x7I7I !%	!6 rG   N)rt   ru   rv   rw   r   r<   r
   r*   r   re   r*  rF   rG   r+   r  r  (  s    a
/
 38_
 !!;<	

 CHo
 ""<=
 23
 34
 uU\\3%67EF
B%rG   r  c                       e Zd ZddZdej
                  j                  fdZdej
                  j                  defdZ	y)"OrtExecutionInfoForAllGraphModulesNc                     i | _         y rc   )execution_info_per_graph_module)rg   s    r+   re   z+OrtExecutionInfoForAllGraphModules.__init__v  s    
  	,rG   r.   c                 t    || j                   vry | j                   |   }|D ]  } |j                  | s|c S  y rc   )r.  r*  )rg   r.   r   
candidates	candidates        r+   &search_reusable_session_execution_infozIOrtExecutionInfoForAllGraphModules.search_reusable_session_execution_info}  sQ     tCCC 99,G
# 	!I%y%%t,  	!
 rG   rp   c                 ~    || j                   vr|g| j                   |<   y | j                   |   j                  |       y rc   )r.  r   )rg   r.   rp   s      r+   cache_session_execution_infoz?OrtExecutionInfoForAllGraphModules.cache_session_execution_info  s=     tCCCBFD00>00>EEdKrG   )r   N)
rt   ru   rv   re   r*   r|   GraphModuler2  r  r4  rF   rG   r+   r,  r,  t  s@    !HH00 L!HH00L8RLrG   r,  r   T)frozenF)is_backward_compatiblec                       e Zd ZU dZdZeee      ed<   	 dZ	e
ed<   	 dZeee      ed<   	 dZe
ed<   	 dZe
ed	<   	 dZed
   ed<   	 dZed   ed<   	 dZeeedgdf         ed<   y)r   aJ  Options for constructing an ``OrtBackend``, the ONNX Runtime
    backend (``"onnxrt"``) for ``torch.compile``.

    Example::

        >>> @torch.compile(
        ...     backend="onnxrt",
        ...     options=torch.onnx._OrtBackendOptions(...),
        ... )
        ... def ort_function(x):
        ...     return x ** x
    Npreferred_execution_providersTinfer_execution_providersdefault_execution_providersFr   use_aot_autogradztorch.onnx.ExportOptionsexport_optionszonnxruntime.SessionOptionsort_session_optionszonnx.ModelProtopre_ort_model_transforms)rt   ru   rv   rw   r9  r   r   r   __annotations__r:  r~   r;  r   r<  r=  r>  r?  r   rF   rG   r+   r   r     s     OS!8H5I,J#KR '+t*mLP(3G*H!IP  %$V!d! <@NH78?UBF">?FV 	 h,-t345 1rG   r   c            	          e Zd ZU dZddee   fdZdej                  j                  de
eeeeef   f      fdZdej                  j                  fdZdej                  j                  dej                  j                  fd	Zdej                  j                  dej                  j                  fd
ZdZeed<   g Zeed       ed<   e	 ddeeeeeef   f      dd fd       Zed        Zed        Zy)r   a	  A backend compiles (sub-)graphs in torch.fx.GraphModule to onnxruntime.InferenceSession calls.

    The compiler entry point is OrtBackend.compile, which
        1. partitions the original graph into supported sub-graphs (type: torch.fx.GraphModule) and unsupported
           sub-graphs.
        2. For each supported sub-graph, it replaces its _wrapped_call function with _ort_accelerated_call.
        3. Inside _ort_accelerated_call, it creates onnxruntime.InferenceSession and calls it to execute the sub-graph.
    Noptionsc                    ddl m} dd l}dd l}dd l}|
t               n|| _        |j                  j                  j                  j                  | j                  j                  |j                  j                         n| j                  j                        | _        |j                  j                  j                  j                  j!                  | j                  j"                        }d d d d d d}t%        ||      | _        i | _        t+               | _        d| _        d| _        t3        |j4                  d      rt6        | _        y t8        | _        y )Nr   r   )getattrz_operator.getitemz_operator.mulz_operator.addz_operator.subFr   )rX   r   r$   r&   +torch.onnx._internal.fx.decomposition_tabler   _optionsonnx	_internal_exporter_legacyResolvedExportOptionsr=  ExportOptions_resolved_onnx_exporter_optionsr|   r   '_create_onnx_supports_op_overload_tableonnx_registryr_   _supported_ops_partitioner_cacher,  _all_ort_execution_info_assert_allclose_to_baselineexecution_countr   r   r  r  r  )rg   rB  r\   r*   r`   ra   s         r+   re   zOrtBackend.__init__  s1   :4:6=o027  JJ  11GG==//7 

((*]]11 	,& zz++..BBjj00>>

  "& "!!
.
 1?QR UW (J'K$,1) 
 t**,=> 2 	 . 	rG   r.   r   c                    d}| j                   j                  rt        | x}r|}nt        |      x}r|}g }g | j                   j                  xs g t        |      | j                   j                  xs
 t               D ]L  }t        |t              r|i f}nt        |t              r|d   |d   i f}|7||vs<|j                  |       N |S )NrF   r3   r   )rF  r:  r   r   r9  r   r;  rH   r   r<   r   r   )rg   r.   r   inferred_epseps_from_argseps_from_graph_moduleselected_epsr   s           r+   _select_epszOrtBackend._select_epsG  s     )+==22 5t <<}<
  -*El*SS&S  5
mm99?R
|$
 mm77O;M;O
 
	(B
 "c""XB&2a5=eR[~"L"8##B'
	( rG   c                 	   ddl }ddlm}m}  | j                  j
                  |g| }|rb|j                  }|j                  }	|j                  }
|j                  }|j                  }|j                  }|j                  }|j                  }n|j                  | j                  j                   |      j#                         }| j                  j$                  r,d| _        t)        |      }d }t+        j,                  ||      }n	  t/        |      j0                  |i |}|j9                  | j                  j                         }|j;                  | j                  j                   |      j#                         }|j#                  || j                  j<                        }|j?                  | j                  j@                  jB                  	      }| jD                  jF                  r#| jD                  jF                  D ]
  } ||        |jI                         }tJ        jL                  jO                  d
d      rtQ        ||        |jR                  || jD                  jT                   | jV                  |g|       }tY        d |jZ                  j\                  D              }	tY        d |jZ                  j^                  D              }
ta        |      }tc        |tX              rta        |      }nta        |f      }tY        d |jZ                  j\                  D              }tY        d |jZ                  j^                  D              }te        ||	||
||||      }| j                  jg                  ||       | xjh                  dz  c_4        tc        |tj        jl                        }|r|fn|}tc        |tX              sJ to        d |D              sJ tq        d       | j#                  ||	|||
||| jD                  j&                  ||
      }ts                | jt                  rhtk        jv                  jx                  jz                  |g|ddi}|r|fn|}t}        ||      D ]%  \  }}tj        j~                  j                  ||       ' |r|d   S |S # t2        $ r t4        j7                  d|       d| _         w xY w)a  This function replaces GraphModule._wrapped_call in compiled model.

        The _wrapped_call is the underlying implementation of forward method. Replacing
        it means we delegate the computation to _ort_acclerated_call and therefore
        onnxruntime.InferenceSession.
        r   N)r   r    Fc                 X    t        | d      rd| j                  v r| j                  d   S | S )Nr   r   )r   r   )r   s    r+   maybe_map_to_meta_valz>OrtBackend._ort_acclerated_call.<locals>.maybe_map_to_meta_val  s-    uf-%5::2E  %zz%00$rG   zFakeTensorProb failed for %s)diagnostic_context)fx_graph_moduleonnxfunction_dispatcher)opset_versionr0   )r.   )path_or_bytessess_options	providersc              3   4   K   | ]  }|j                     y wrc   rO   r   inputs     r+   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     Ou

O   c              3   4   K   | ]  }|j                     y wrc   rO   r   r   s     r+   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s      S Srg  c              3       K   | ]  }|  y wrc   rF   re  s     r+   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     %Pe%P   c              3       K   | ]  }|  y wrc   rF   ri  s     r+   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s     &T&v&Trk  r!  r3   c              3   z   K   | ]3  }t        |t        j                  t        j                  t        f       5 y wrc   )r   r*   r   r   r   )r   elems     r+   r   z2OrtBackend._ort_acclerated_call.<locals>.<genexpr>  s.      
 tellELL#>?
s   9;$run_onnx_session_with_ortvaluevectorexecutoraten)Ar   r(   r   r    rQ  r2  r  r   r   r   r  r   r   r  MovePlaceholderToFrontrL  r]  r  dynamic_shapesr   r   r   tree_mapr   	propagate	Exceptionro   rs   FxOnnxInterpreterInsertTypePromotionr_  to_model_protorN  r`  rF  r?  SerializeToStringr7   r8   r9   rC   InferenceSessionr>  rY  r   r=   rf  r   r   r   r  r4  rS  r*   r   allrP   rS   rR  _primsrp  executer  testingassert_close)rg   r.   r   kwargsr   r   r    !cached_execution_info_per_sessiononnx_sessionr   r   r   r  r   r   prim_outputsextracted_outputsr\  fx_interpreterexported
onnx_model	transformonnx_model_bytesexecution_info_per_sessionis_single_tensor_outputr   onnx_outputsbaseline_outputsnormalized_baseline_ouptutsr  baseline_outputs                                  r+   _ort_acclerated_callzOrtBackend._ort_acclerated_callh  s    	G PD((OO# 	*
 -<DDL;GGK<IIL A S S!B!U!U=KKM>MMN<LLL
 "8844GG ce  33BB*/'$A,$O!%  '//)+< #I>,#?#I#I$!'$L 1BB#'#G#G#Z#Z C N "5544GGce 
 &)) ,(,(L(L(d(d * H
 "00"BBPP^^ 1 J }}55!%!G!G *Ij)*  *;;=zz~~0$7 !!1M 8;77.!]]>>*$**<?$?L  O
8H8H8N8NOOK  S:;K;K;R;R SSL-d3M ,.!2<!@!2L?!C %%P9I9I9O9O%P P!&&TJ<L<L<S<S&T!T)C$'"3)#5+- ,	*& ((EE8 	!
 #-\5<<"H6\OL 	  15999 
/
 
 	
 

 	?@xx#MM,,#
 	,,$||44<< # .4  (?!#DT ( 1491 I,_ **;HI #:|AK|KW ! NN#A<P /4D+ s   R (R?c                 ~   ddl m} || j                  v r| j                  |   }|S |} ||| j                  d      }|j	                         }|| j                  |<   |j
                  j                  D ]H  }|j                  dk(  sd|j                  v s"t        ||j                        }| j                  |_        J |S )Nr   )CapabilityBasedPartitionerT)allows_single_node_partitioncall_modulefused_)!torch.fx.passes.infra.partitionerr  rP  rO  partition_and_fuser=   r   rm   rI   rD  r  _wrapped_call)	rg   r.   r   r  partitioned_prim_graph_moduleprim_graph_modulepartitionerrj   fused_modules	            r+   compilezOrtBackend.compile1  s    
 	Q, 4222,0,C,CL,Q)0 -,- !-4!##-1K
 -8,J,J,L)4QD##L1 6;;AA K 77m+DII0E#*+H$))#TL 261J1JL.K -,rG   c                     | j                   j                  r<ddlm} ddlm}   || j                  || j                  j                        ||      S | j                  ||      S )zIf ``OrtBackendOptions.use_aot_autograd`` is ``True``, the `auto_autograd` compiler
        will be invoked, wrapping this ``OrtBackend`` instance's ``compile`` method. Otherwise,
        the ``compile`` method is invoked directly.r   )#min_cut_rematerialization_partition)aot_autograd)fw_compilerpartition_fndecompositions)	rF  r<  functorch.compiler  torch._dynamo.backends.commonr  r  rL  r   )rg   r.   r   r  r  s        r+   __call__zOrtBackend.__call__g  s`     ==))MB< LL@#CCWW D	" " ||L$//rG      %_OrtBackend__instance_cache_max_count_OrtBackend__instance_cachec                     dt         dt         fdt         t               st        d
i  xs i  t         fdt        j                  D        d      }|{t        t        j                        t        j                  k  s'J dt        j                   dt         dt         d	       t        j                  j                  t               x}       |S )a  Returns a possibly cached instance of an ``OrtBackend``. If an existing
        backend was created previously through this function with the same options,
        it will be returned. Otherwise a new backend will be created, cached, and
        returned.

        Note: if ``options`` sets ``ort_session_options``, a new ``OrtBackend``
        will always be returned, since ``onnxruntime.SessionOptions`` cannot
        participate in caching.abc                 6   | j                   |j                   k7  s}| j                  |j                  k7  sd| j                  |j                  k7  sK| j                  |j                  k7  s2| j                  |j                  k7  s| j
                  |j
                  k7  ry| j                  |j                  y| j                  |j                  u ry| j                  |j                  | j                  j                  |j                  j                  k(  xr | j                  j                  |j                  j                  k(  xrZ | j                  j                  |j                  j                  u xr, | j                  j                  |j                  j                  u S y)NFT)r9  r:  r;  r   r<  r?  r>  r=  rs  diagnostic_optionsrN  fake_context)r  r  s     r+   reusablez<OrtBackend.get_cached_instance_for_options.<locals>.reusable  sl   //13R3RR..!2M2MM00A4Q4QQ''1+?+??%%););;--1K1KK $$0A4I4I4U1#3#33 +0@0@0L$$33q7G7G7V7VV W((;;''::;W ((66!:J:J:X:XXW ((559I9I9V9VV rG   c              3   J   K   | ]  } |j                         s|  y wrc   )rF  )r   r  rB  r  s     r+   r   z=OrtBackend.get_cached_instance_for_options.<locals>.<genexpr>  s     U1x

G7TQUs   ##NzNo more than z instances of z allowed. Please instantiate `z` explicitly to pass to `torch.compile`. See https://github.com/pytorch/pytorch/pull/107973#discussion_r1306144795 for discussion.rF   )r   r   nextr   r  r   r  r   )rB  backendr  s   ` @r+   get_cached_instance_for_optionsz*OrtBackend.get_cached_instance_for_options|  s    !	) !	.? !	F '#45':7=b:GU
33U

 ?J//0:3X3XX  
 E EFn,<ZL I""X ''..*W:M/MwNrG   c                  @    t         j                  j                          y rc   )r   r  clearrF   rG   r+   clear_cached_instancesz!OrtBackend.clear_cached_instances  s    ##))+rG   c                  4    t        t        j                        S rc   )r   r   r  rF   rG   r+   get_cached_instanceszOrtBackend.get_cached_instances  s    Z0011rG   rc   )rt   ru   rv   rw   r   r   re   r*   r|   r5  r   r   r<   r   r   rY  r  r  r  r  r   r@  r  liststaticmethodr
   r  r  r  rF   rG   r+   r   r     sM   V
): ; V
p!HH00	%WS#X../	0BGL1E1E GLR4-EHH$8$8 4-588CWCW 4-l0!HH000			0$ )*)24eD./4IMC% 17383D DEFC	C CJ , , 2 2rG   r   )rB  rB  c                8    t        j                  |      | |      S rc   )r   r  )r.   r   rB  s      r+   r   r     s     55g>|TRRrG   rc   )[dataclassesr"   loggingr7   collections.abcr   r   typingr   r   r   r   r	   r
   typing_extensionsr   r*   torch._C
torch._opstorch._prims.executortorch.fxtorch._subclasses.fake_tensorr   torch.fx._compatibilityr    torch.fx.passes.fake_tensor_propr    torch.fx.passes.operator_supportr   torch.fx.passes.tools_commonr   torch.utilsr   rG  r   rX   r   r\   r$   r%   r&   r'   rE  torch.onnx._internal.fx.passesr   r~   r@  __all__r   r,   ry   r<   r   bytesr|   r5  rC   rH   rP   rS   r]   	getLoggerrt   ro   r_   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  	dataclassr,  r   r   r   r   rF   rG   r+   <module>r     sY      	 - G G '      4 1 ; < :  60+6) #'$ &.T .b &( DcN ' IM'/0D0D'E2$HSM $)3 )$
@c 
@ 
		8	$/ /d3UXX-A-A 3d 3*
E#s(O 
uxx/C/C cSVh K0D0D K K8ehh.B.B 8uSRUX 8 X5c? XuS#X X$%2LL%,,U^^UEMMSWW	
 		%2  !%2P5<<$%056K0L
5<<*J 5<< )	) &) \\)@LL	 	LL	
8N
(NsCxN %,,#$N ./	N
 S/N 5<<$%N /0N N 78N #LL%,,U^^UEMMSWW	
 		N  5sE4/0#56!Nb#
(#sCx# %,,#$# ./	#
 S/# 5<<$%# /0# # 78# #LL%,,U^^UEMMSWW	
 		#  5sE4/0#56!#LI IX L L LB #(U3S8I3I-J(J"K i K	 d#e,<1 <1 - $<1~ e,c2 c2 -c2L e,
 FJ	S((&&S e-wsCx/@@AB	S -SrG   