
    VhdA                     4   d dl Z d dlZd dlmZmZmZ d dlZd dlZd dlZd dl	m
Z
 d dlZd dlmZmZ d dlZd dlZ ej"                  e      Zd Zd Z e       Zd Zeee ed      eefZdZd	 Z  G d
 de      Z!ddddZ"d Z#d Z$d Z%d Z&dZ'dZ(d Z)d Z*d Z+y)    N)Any
NamedTupleOptional)NamedTemporaryFile)_frames_fmt_block_extrac                      dfd}t        j                  |        fdt        j                  j	                         fd}|S )NTc                  
    d y )NF )enableds   G/home/dcms/DCMS/lib/python3.12/site-packages/torch/utils/viz/_cycles.pydisablez observe_garbage.<locals>.disable   s
         c                     sy | dk(  r$t        j                  t         j                         y | dk(  r6t        j                         dgfd}t        j
                  |       y y )NstartstopFc                     d   sdd<   nt        j                         d	 d   dk7  rt        j                           t        j                         t        j                  j                          t        j                  d       t        j                  j                         }t        j                          t        j                  j                         }||k7  rt        j                  d||z
         d | i |S y # dw xY w)Nr   TF
generation   z.CUDA Memory changed during GC, %d bytes freed.)sys
setprofilegccollectgarbageclear	set_debugtorchcudamemory_allocatedloggerwarning)	argskwargsbeforeafterr   infoobserver
orig_traceself_returns	       r   
do_collectz8observe_garbage.<locals>.gc_callback.<locals>.do_collect#   s    "1~%)KNNN:.#G'  -2JJL ,

((* Q!&!<!<!>

 %

 ; ; =!U?"NN+[]cfk]kl"&)%t6v66 * #'s   CD D	)r   r   DEBUG_SAVEALLr   
getprofiler   )phaser&   r*   r(   r)   r   r'   s    ` @@r   gc_callbackz$observe_garbage.<locals>.gc_callback   s[    GLL))*f_)J 'K7 7: NN:&C r   c                  D    t         j                  j                          y N)r   	callbacksremove)r.   s   r   r2   zobserve_garbage.<locals>.removeE   s    
K(r   )atexitregisterr   r1   append)r'   r   r2   r   r.   s   `  @@r   observe_garbager6      s?    G
 OOG''R LL$)Mr   c                  D    dd} t         |        j                  d         S )Nc                       fdS )Nc                       S r0   r   xs   r   <lambda>z+_get_cell_type.<locals>.f.<locals>.<lambda>Z   s    q r   r   r:   s   `r   fz_get_cell_type.<locals>.fY   s	    r   r   r0   )type__closure__)r=   s    r   _get_cell_typer@   X   s    "##r   c                     i fd fdfd}fd} fd} fd} fd}fd} fd	} fd
}fd}	t         |t        |t        |t        |t        |t
        j                  |t
        j                  |t        |t
        j                  |t        j                  |t
        j                  |	i}
t               j                  D ]  }||
v s |
|             dd       t         t              r d       S )a%  
    Return known information about references held by the given object.

    Returns a mapping from referents to lists of descriptions.  Note that there
    may be more than one edge leading to any particular referent; hence the
    need for a list.  Descriptions are currently strings.

    c                 Z    j                  t        |      g       j                  |        y r0   )
setdefaultidr5   )nameobj
referencess     r   add_referencez+annotated_references.<locals>.add_referencej   s"    bgr*11$7r   c                  T    | D ]"  }t        |      s |t        |             $ y r0   )hasattrgetattr)attrsattrrH   rF   s     r   	add_attrsz'annotated_references.<locals>.add_attrsm   s-     	8DsD!dGC$67	8r   c                  6    	   d       y # t         $ r Y y w xY w)Ncell_contents)
ValueErrorrN   s   r   add_cell_referencesz1annotated_references.<locals>.add_cell_referencesr   s$    	o& 	
 	s    	c                  &      ddddddddd		       y )
N__defaults__r?   __globals____code____name__
__module____doc____qualname____annotations____kwdefaults__r   rR   s   r   add_function_referencesz5annotated_references.<locals>.add_function_references|   s'    .!#"		$r   c                  F    t              D ]  \  } } d|  d|        y )N[])	enumerate)positionitemrH   rF   s     r   add_sequence_referencesz5annotated_references.<locals>.add_sequence_references   s,    'n 	1NHdAhZq/40	1r   c                  t    j                         D ]$  \  } } d|         dt        |        d|       & y )Nkeyr_   r`   )itemsrepr)rf   valuerH   rF   s     r   add_dict_referencesz1annotated_references.<locals>.add_dict_references   s=    ))+ 	3JC%%Ad3i[*E2	3r   c                  &    D ]  }  d|         y )Nelementr   )eltrH   rF   s    r   add_set_referencesz0annotated_references.<locals>.add_set_references   s     	*C)S)	*r   c                        ddd       y )N__self____func__im_classr   rR   s   r   add_bound_method_referencesz9annotated_references.<locals>.add_bound_method_references   s    *j*5r   c                      t              t        j                  u r3t        j                        } t        |       dk(  r| d   } d|       y y y )N   r   __callback__)r>   weakrefrefr   get_referentslen)	referentstargetrH   rF   s     r   add_weakref_referencesz4annotated_references.<locals>.add_weakref_references   sN     9#((-I9~""1nf5 # $r   c                      j                   }  dddddd       t        |       t        u r/j                   j                         D ]  \  }} d| |        y y )Nf_backf_code
f_builtins	f_globalsf_tracef_localszlocal )r   r>   dictrg   )r   rE   localrN   rH   rF   s      r   add_frame_referencesz2annotated_references.<locals>.add_frame_references   sa    <<(HlKJW >T!"||113 6etfou56 "r   c                        ddd       y )N__objclass__rX   __doc__r   rR   s   r    add_getset_descriptor_referencesz>annotated_references.<locals>.add_getset_descriptor_references   s    .*i8r   __dict__	__class____mro__)tuplelistr   set	frozensettypesFunctionType	FrameTypeCellType
MethodTyperw   rx   GetSetDescriptorTyper>   r   
isinstance)rF   rS   r]   rd   rj   rn   rs   r}   r   r   type_based_referencestype_rN   rH   rG   s   `           @@@r   annotated_referencesr   _   s     (*J88

$13
*6669 	&%!%3-%5+""$D c"" +))(!%(*+ j+&#t)r       c                    d }t        | t              rt        |       S t        |       j                  dk(  rd| j                   S t        | t
        j                        r	 | j                  j                  }d| S t        | t              rd ||        dS t        | t              rd ||        d	S t        | t              rd
t        |        dS t        | t
        j                        rd| j                   S t        | t              rd| j                   S t        | t        j                         r |        }|ydt#        |      dS t        | t
        j$                        rJ| j&                  j(                  }t        |      t*        kD  rd|t*        dz
   d z   }d| d| j,                   S dt        |       j.                   dt        |       j                   S # t        $ r d}Y ww xY w)zx
    Return a string to be used for Graphviz nodes.

    The string should be short but as informative as possible.
    c                     dj                  d t        t        d      |       D              }t        |       dkD  r| dt        |       dz
   }|S )N,c              3      K   | ]7  \  }}t        |t              rt        |      nt        |      j                   9 y wr0   )r   
BASE_TYPESrh   r>   rX   ).0ir;   s      r   	<genexpr>z=object_annotation.<locals>.format_sequence.<locals>.<genexpr>   s3     rW[WXZ[:a#<Q$q'BRBRRrs   =?   z, ...)joinziprangerz   )rF   bodys     r   format_sequencez*object_annotation.<locals>.format_sequence   sN    xxr_bchijckmp_qrrs8a<V5SA/Dr   functionz	function
z<anonymous>zinstancemethod
r_   r`   ()zdict[zmodule
ztype
Nzweakref (dead referent)zweakref to id 0xr;   z...   zframe
:zobject
.)r   r   rh   r>   rX   r   r   rq   __qualname__AttributeErrorr   r   r   rz   
ModuleTyperw   rx   rD   r   r   co_filenameFRAME_FILENAME_LIMITf_linenorY   )rF   r   	func_namereferentfilenames        r   object_annotationr      s    #z"CyCyZ'CLL>**	C))	*	&11I ")--	C	?3'(**	C	?3'(**	C	s3xj""	C))	*#,,((	C	~&&	C	%5,%bl1%566	C	)::))x=//x*>*B(C(DEEH
!CLL>22$s)../qc1C1C0DEE3  	&%I	&s   !G   G/.G/c                   L    e Zd ZU eed<   ee   ed<   eed<   eeee	f      ed<   y)Nodelabelcontextroot
referrentsN)
rX   rY   r   strr[   r   boolr   r   intr   r   r   r   r     s)    Jc]
JU38_%%r   r   )r   filterc          
         |
t               }|t        }| D cg c]  }t        |t        j                        r|! } }| D cg c]%  }t        t        |       ||       ||      g       ' }}| D cg c]  }g  }}t        |       D ci c]  \  }}t        |      | }}}| D ]  }|t        |         }||   }	t        |      }
t        j                  |      D ]m  }t        |      }|j                  |d       }|#|
j                  |dg      }||   j                  |       |D ]  }|	j                  j                  ||f       ! o  t        |      D cg c]  \  }}|j                  s| }}}t!               }|r?|j#                         }||v r|j%                  |       ||   }|j'                  |       |r?i }g }t        |      D ])  \  }}||v st)        |      ||<   |j                  |       + |D ]7  }|j                  D cg c]  \  }}||v r|||   f c}}|j                  d d  9 |S c c}w c c}w c c}w c c}}w c c}}w c c}}w )N?)cuda_allocation_contextis_cuda_tensorr   rw   
ProxyTypesr   r   ra   rD   r   r   ry   getr5   r   r   r   popaddextendrz   )objectsr   r   rF   nodesnode_referrersr   
id_to_nodefidxr=   rG   	referrentridtidxlabelsr   n	to_searchto_keepidx	referrersid_to_filtered_idfiltereds                          r   create_graphr     sv   )+~%QsZW=O=O-PsQGQT[\ST#C('#,sRH\E\5<&=cr&=N&=+4W+=>C"S'1*>J> 3"S'"$K)#.
))#. 	3IY-C>>#t,D|^^C#/F4 ''- 3##UDM23	3	3  )/:tq!166:I:eG
mmo'>C"3'	#  )+H%  1<#&'8#9a OOA  8/0||8+s!%66 "#4S#9: 8Q8 OM R\&=> ; 8s.   H=H=*I3	III2IIc                 ,    t        j                  |       S r0   )jsondumps)r   s    r   escaper   5  s    ::a=r   c                     t        | t        j                        xr3 | j                  xr% t        | t        j                  j
                         S r0   )r   r   Tensoris_cuda_subclasses
FakeTensor)rF   s    r   r   r   9  s8    c5<<(nS[[nCQVQbQbQmQmAn=nnr   c                      t         j                  j                  j                         } i | d   D ]4  }|d   }|d   D ]%  }|d   dk(  rt	        |      \  }}||<   ||d   z  }' 6 fd}|S )Nsegmentsaddressblocksstateactive_allocatedsizec                     t        |       rM| j                         j                         }j                  |      }|dj	                  t        |d            S y )N
T)full_filename)r   untyped_storagedata_ptrr   r   r   )rF   addrframesaddr_to_frames      r   object_contextz/cuda_allocation_context.<locals>.object_contextG  sQ    #&&(113D"&&t,F!yyV4!HIIr   )r   r   memory	_snapshotr   )snapshotsegr   blkr   
_real_sizer   r   s          @r   r   r   <  s    zz  **,HM
#  9~x= 	 C7|11%1#%6"
&,d#CKD		   r   c                 ~   g d}t        |       D ]@  \  }}|j                  | dt        |j                         d|j                  rdnd d       B t        |       D ]<  \  }}|j
                  D ](  \  }}|j                  | d| dt        |       d	       * > |j                  d
       dj                  |      S )N)zdigraph GraphName {znode [shape=rect];zrankdir=LR;z [label=z, color=redblackz];z -> z
 [label = r`   z}
r   )ra   r5   r   r   r   r   r   )r   linesr   r   r=   r   js          r   to_dotr  P  s    HE%  _1s(6!''?"38affURY;ZZ\]^_ %  B1 	BHE1LLA3d1#Zua@A	BB 
LL99Ur   a  
<!DOCTYPE html>
<html>
<head>
  <style>
    body {
      margin: 0;
      padding: 0;
      overflow: hidden;
    }

    #container {
      display: flex;
      flex-direction: column;
      height: 100vh;
    }

    #main {
      flex: 2;
      height: 60vh;
      overflow: clip;
    }

    #preContainer {
      flex: 1;
      height: 40vh;
      overflow: auto;
    }

    pre {
      margin: 0;
      padding: 10px;
    }
  </style>
</head>
<body>
  <div id="container">
    <div id="main">
    </div>
    <div id="preContainer">
      <pre id="stacktrace">Mouse over tensor objects to see where they were allocated.</pre>
    </div>
  </div>
<script src='https://cdnjs.cloudflare.com/ajax/libs/viz.js/1.8.0/viz-lite.js'></script>
<script>
let dot = $DOT
let image = Viz(dot, {format: 'svg', 'totalMemory': 1024*1024*1024});
let main = document.getElementById('main')
main.innerHTML = image
let svg = main.firstElementChild
// Panning and zooming logic
let isPanning = false;
let startX, startY;
let viewBox = { x: 0, y: 0, width: parseFloat(svg.getAttribute('width')), height: parseFloat(svg.getAttribute('height')) };
svg.removeAttribute('width');
svg.removeAttribute('height');
function updateViewBox() {
    svg.setAttribute('viewBox', `${viewBox.x} ${viewBox.y} ${viewBox.width} ${viewBox.height}`);
}
updateViewBox()
svg.setAttribute('preserveAspectRatio', 'xMidYMid meet');
svg.addEventListener('mousedown', function(e) {
    isPanning = true;
    startX = e.clientX;
    startY = e.clientY;
});
svg.addEventListener('mousemove', function(e) {
    if (!isPanning) return;
    const dx = (e.clientX - startX) * (viewBox.width / svg.clientWidth);
    const dy = (e.clientY - startY) * (viewBox.height / svg.clientHeight);
    viewBox.x -= dx;
    viewBox.y -= dy;
    startX = e.clientX;
    startY = e.clientY;
    updateViewBox();
});
svg.addEventListener('mouseup', function() {
    isPanning = false;
});
svg.addEventListener('mouseleave', function() {
    isPanning = false;
});
svg.addEventListener('wheel', function(e) {
    e.preventDefault();
    const zoomFactor = 0.1;
    const zoomAmount = e.deltaY > 0 ? 1 + zoomFactor : 1 - zoomFactor;
    // Calculate mouse position relative to the SVG
    const rect = svg.getBoundingClientRect();
    const mouseX = e.clientX - rect.left;
    const mouseY = e.clientY - rect.top;
    const mouseXRel = mouseX / svg.clientWidth;
    const mouseYRel = mouseY / svg.clientHeight;
    // Adjust viewBox to zoom around the mouse position
    const newWidth = viewBox.width * zoomAmount;
    const newHeight = viewBox.height * zoomAmount;
    viewBox.x += (viewBox.width - newWidth) * mouseXRel;
    viewBox.y += (viewBox.height - newHeight) * mouseYRel;
    viewBox.width = newWidth;
    viewBox.height = newHeight;
    updateViewBox();
});
$LISTENERS
</script>
</body>
</html>
z
document.getElementById('node{id}').addEventListener('mouseover', function(event) {{
  document.getElementById("stacktrace").textContent = {stack}
}})
c           	         g }t        |       D ]h  \  }}|j                  t        j                  t	        |dz         t        |j                   d|j                               }|j                  |       j t        |       }t        j                  dt        |            j                  ddj                  |            S )Nru   z:
)rD   stackz$DOTz
$LISTENERSr   )ra   r   _listener_templateformatr   r   r   r5   r  	_templatereplacerh   r   )r   	listenersr   r   sdots         r   to_htmlr    s    I%  199%%QU6QWWISQRQZQZP[B\;]%^	
 -CVT#Y/77diiPYFZ[[r   c                 x     t         j                  j                  j                  d        fd}t	        |      S )Ni )max_entriesc                     | rCt        d | D              st        j                  d       y  t        t	        |                    y y )Nc              3   2   K   | ]  }t        |        y wr0   )r   )r   rF   s     r   r   z:observe_tensor_cycles.<locals>.observer.<locals>.<genexpr>  s     >s~c*>s   z No CUDA Tensors found in garbage)anyr    r&   r  r   )r   callbacks    r   r'   z'observe_tensor_cycles.<locals>.observer  s;    >g>>>?W\'234	 r   )r   r   r   _record_memory_historyr6   )r  r'   s   ` r   observe_tensor_cyclesr    s0    	JJ,,,@5 8$$r   c                  H    t         j                  d       d } t        |       S )a  
    Install a warning that reports whenever a cycle that is holding CUDA memory is observed.

    The warning produces an .html file that visualizes the cycle,
    and links it to the stack frame that allocted the CUDA tensor.

    Reference cycles are freed by the cycle collector rather than being cleaned up
    when the objects in the cycle first become unreachable. If a cycle points to a tensor,
    the CUDA memory for that tensor will not be freed until garbage collection runs.
    Accumulation of CUDA allocations can lead to out of memory errors (OOMs), as well as
    non-deterministic allocation behavior which is harder to debug.
    z2Watching Python reference cycles for CUDA Tensors.c                     t        ddd      5 }|j                  |        t        j                  d|j                         d d d        y # 1 sw Y   y xY w)Nwz.htmlF)suffixdeletezDReference cycle includes a CUDA Tensor see visualization of cycle %s)r   writer    r!   rE   )htmlr=   s     r   write_and_logz)warn_tensor_cycles.<locals>.write_and_log  sJ    GEB 	kaGGDMNNacdcicij	k 	k 	ks   2A

A)r    r&   r  )r  s    r   warn_tensor_cyclesr    s$     KKDEk !//r   ),r   r   typingr   r   r   r   rw   r   tempfiler   r   torch.cuda._memory_vizr   r   r3   logging	getLoggerrX   r    r6   r@   r   r   r   floatcomplexr>   r   bytesr   r   r   r   r   r   r   r   r  r	  r  r  r  r  r   r   r   <module>r(     s    	 
 , ,    '  <  			8	$8R$
 gZ 5'4:sE:
 .Fd&: & &*$ ,\o(	i	T 
\	%0r   