
    BVhi	                     R    d Z ddlZddlZddlZddlZddlZddlmZ  G d de      Z	y)z6Watchdog that monitors activity of ClusterCoordinator.    N)loggingc                   D    e Zd ZdZdej
                  dfdZd Zd Zd Z	y)WatchDogzJA class to dump stack traces if no activity happens in ClusterCoordinator.Nc                    t         j                  j                  dd      j                         rt	        t         j                  d         }|| _        t        j                         | _        || _        || _	        d| _
        |dkD  rBt        j                  | j                  dd      | _        | j                  j                          y y )N(TF_CLUSTER_COORDINATOR_WATCH_DOG_TIMEOUT Fr   r   T)targetnamedaemon)osenvironget	isnumericint_timeouttime_last_activity_time_traceback_file_on_triggered_stopped	threadingThread_watchdog_function_watchdog_threadstart)selftimeouttraceback_fileon_triggereds       a/home/dcms/DCMS/lib/python3.12/site-packages/tensorflow/python/distribute/coordinator/watchdog.py__init__zWatchDog.__init__   s    	zz~~@#)+&BJJIJKgDM#yy{D)D%DDM{'..((z$Hd
!!#     c                     d| _         y )NT)r   r   s    r!   stopzWatchDog.stop*   s	    DMr#   c                    t        j                  d| j                         | j                  st	        j
                  | j                  dz         t	        j                         }|| j                  z
  | j                  k\  r~t        j                  d| j                         | j                  r| j                          t        j                  | j                         | j                  j                  d       || _        | j                  syy)zThe watchdog thread.z(Starting watchdog thread with timeout %rg      $@zHNo activity for ClusterCoordinator for %r seconds. Dumping stack traces.)filez==== End of stack traces ====
N)r   infor   r   r   sleepr   warningr   faulthandlerdump_tracebackr   write)r   current_times     r!   r   zWatchDog._watchdog_function-   s    LL;T]]Kmm
jj%&YY[l	00	0DMM	A$%)]]	4 



##)=)=>""#DE#/  mmr#   c                 V    | j                   dkD  rt        j                         | _        y y )Nr   )r   r   r   r%   s    r!   report_closure_donezWatchDog.report_closure_done=   s!    }}q!%d r#   )
__name__
__module____qualname____doc__sysstdoutr"   r&   r   r1    r#   r!   r   r      s$    R

 $0 -r#   r   )
r5   r,   r   r6   r   r   abslr   objectr   r8   r#   r!   <module>r;      s(    =  	 
   &-v &-r#   