
    BVh                         d Z ddlZddlZddlZddlZddlmZ ddlm	Z ddl
mZ  G d dej                        Z G d d	ej                        Zy)
z@Implementation of the SessionRunHook for preemptible Cloud TPUs.    N)tpu_cluster_resolver)
tf_logging)session_run_hookc                   "    e Zd ZdZd Zd Zd Zy)CloudTPUPreemptedHooka  The SessionRunHook for preemptible Cloud TPUs.

  This is an implementation of SessionRunHook for the pre-emptible Google Cloud
  TPU service. It attempts to close the session if the TPU is preempted, and
  exits the coordinator process if the session cannot be closed.
  c                     || _         y N)_cluster)selfclusters     T/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/tpu/preempted_hook.py__init__zCloudTPUPreemptedHook.__init__#   s	    DM    c                     t        j                         r6t        | j                  |      | _        | j                  j                          y y r	   )r   is_running_in_gce_TPUPollingThreadr
   _tpu_pollerstart)r   sessioncoords      r   after_create_sessionz*CloudTPUPreemptedHook.after_create_session&   s8    --/*4=='Bd
 0r   c                 8    | j                   j                          y r	   )r   stop)r   r   s     r   endzCloudTPUPreemptedHook.end+   s    r   N)__name__
__module____qualname____doc__r   r   r    r   r   r   r      s    
r   r   c                   .     e Zd ZdZ fdZd Zd Z xZS )r   a%  A thread that polls the state of a TPU node.

  When the node transitions into a TERMINAL state (PREEMPTED, TERMINATED)
  that's considered as not recoverable by the underlying infrastructure,
  it attempts to close the session, and exits the entire process if the
  session.close() stucks.
  c                     t         t        |           d| _        d| _        d| _        || _        || _        d| _        dD ]4  }t        j                  |      j                  t        j                         6 y )NTF   )zgoogleapiclient.discoveryzoauth2client.client)superr   r   daemon_running_session_closedr
   _session	_interval_logging	getLoggersetLevelWARNING)r   r   r   name	__class__s       r   r   z_TPUPollingThread.__init__8   sm    	
T+-DKDM DDMDMDN E :''(8(89:r   c                 @    d| _         d| _        | j                          y )NFT)r%   r&   join)r   s    r   r   z_TPUPollingThread.stopF   s    DMDIIKr   c                    t        j                         st        j                  d       d| _        y | j                  r| j
                  j                  j                         }|sbt        j                  d| j
                  j                  | j
                  j                  j                                t        j                  d       t        j                  | j                         | j                  ry y )Nz@TPUPollingThread is running in a non-GCE environment, exiting...Fz)TPUPollingThread found TPU %s in state %s   )r   r   loggingwarningr%   r
   _cloud_tpu_clientrecoverable_tpustateos_exittimesleepr(   )r   r6   s     r   runz_TPUPollingThread.runK   s    113oo
LNdm
--MM33??Ak7MM ? ? E E G	I 	
jj  --r   )r   r   r   r   r   r   r=   __classcell__)r.   s   @r   r   r   /   s    :
!r   r   )r   r3   r)   r9   	threadingr;   -tensorflow.python.distribute.cluster_resolverr   tensorflow.python.platformr   tensorflow.python.trainingr   SessionRunHookr   Threadr   r   r   r   <module>rE      sE    G  	   N < 7,;; (*!	(( *!r   