Vh*(ddlZddlZddlZddlZddlmZmZddlmZddl m Z m Z m Z ddl mZmZmZddlmZmZddlmZmZmZmZdd lmZmZdd lmZmZej>e Z!d Z"Gd d eZ#dedefdZ$dedefdZ%dede&e#effdZ'y)N) b64decode b64encode) timedelta)AnycastOptional) FileStoreStoreTCPStore)construct_and_record_rdzv_event NodeState)RendezvousConnectionErrorRendezvousErrorRendezvousParametersRendezvousStateError)RendezvousBackendToken)_matches_machine_hostnameparse_rendezvous_endpointirc eZdZUdZdZeed<eed<dededdfd Ze defd Z de e e effd Z dd e d e ede e e eeffdZdedefdZde de e e effdZy)C10dRendezvousBackendzRepresents a C10d-backed rendezvous backend. Args: store: The :py:class:`torch.distributed.Store` instance to use to communicate with the C10d store. run_id: The run id of the rendezvous. Y2FuaW1hZGFt_store_keystorerun_idreturnNc|s td||_d|z|_|jd|jd|jy)Nz&The run id must be a non-empty string.ztorch.rendezvous. compare_set) ValueErrorrr _call_store_NULL_SENTINEL)selfrrs l/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/elastic/rendezvous/c10d_rendezvous_backend.py__init__zC10dRendezvousBackend.__init__4sDEF F '&0   2t7J7JKcy)See base class.c10d)r%s r&namezC10dRendezvousBackend.nameDsr(c\|jd|j}|j|S)r*get)r#r _decode_state)r% base64_states r& get_statezC10dRendezvousBackend.get_stateIs)"..udii@ !!,//r(statetokencHt|j}|r=t|ts|j }| g|d}|Sy|j}n |j }|j d|j||}|j|}|y|\}} || ||k(fS)r*NFr ) rdecode isinstancebytesr2r$r#rr0) r%r3r4base64_state_strresulttmpr1state_token_pair new_state new_tokens r& set_statezC10dRendezvousBackend.set_stateOs!*% 0 7 7 9 eU+)%(6.5.CJLLNE''E".. 499e-=  --l;  #/ 9 )Y%%777r(store_opc t|j||i|S#tttf$r}t d|d}~wwxYw)NMThe connection to the C10d store has failed. See inner exception for details.)getattrrr" RuntimeError TimeoutErrorr)r%r@argskwargsexcs r&r#z!C10dRendezvousBackend._call_storessN 174;;14B6B BL,7 +_  sA >Ar1c||jjk(ry t|}||fS#tj$r}t d|d}~wwxYw)Nz=The state object is corrupt. See inner exception for details.)r$encoderbinasciiErrorr)r%r1r3rHs r&r0z#C10dRendezvousBackend._decode_state{sa 4..557 7 l+E l"" ~~ &O  s /A AA)N)__name__ __module__ __qualname____doc__r$r __annotations__strr'propertyr-rtupler8rr2boolr?rr#r0r,r(r&rr#s$N M ILeLSLTL c08E%,$7806:"8"8#+E?"8 %ud*+ ,"8HCS #% #HU5%<=P4Q #r(rparamsrc <t|jt\}}|jd}||}n t |}t t |jdd}|dkr td|dfD]t} t|||dt|  }|rSd tjd }t|j|tj t"j%||SS#tt&t(f$r} |r| t+d| Yd} ~ d} ~ wwxYw)N) default_portis_host read_timeout<rz,The read timeout must be a positive integer.FT)seconds) is_master multi_tenanttimeoutzProcess z5 hosts the TCP store for the C10d rendezvous backend.)rmessage node_staterB)rendpoint DEFAULT_PORT get_as_boolrrint get_as_intr"r rosgetpidr rr INITloggerinforDrEr) rVhostport cfg_is_hostrYrZ is_serverrmsgrHs r&_create_tcp_storerqs2*6??VJD$$$Y/K,D1V..~rBCLqGHHu%  #!!,7 E  -bc/!==#).. C  L=< LL,7  7/c!8 s8A/C--DDDc|jr |j}n tj\}} t |}|S#t$r}t d|d}~wwxYw#t tf$r}td|d}~wwxYw)NzMThe file creation for C10d store has failed. See inner exception for details.rB) rbtempfilemkstempOSErrorrr r"rDr)rVpath_rHrs r&_create_file_storerxs  &&(GAt $ L !_    %' [  s.A A A AAA=, A88A=c|jddjj} |dk(r t|}n|dk(r t |}n t dt ||j}||fS#t$rM}tt|jdt||jtjd}~wwxYw)a Create a new :py:class:`C10dRendezvousBackend` from the specified parameters. +--------------+-----------------------------------------------------------+ | Parameter | Description | +==============+===========================================================+ | store_type | The type of the C10d store. The currently supported types | | | are "tcp" and "file" which correspond to | | | :py:class:`torch.distributed.TCPStore` and | | | :py:class:`torch.distributed.FileStore`, respectively. | | | Defaults to "tcp". | +--------------+-----------------------------------------------------------+ | read_timeout | The read timeout, in seconds, for store operations. | | | Defaults to 60 seconds. | | | | | | Note this only applies to | | | :py:class:`torch.distributed.TCPStore`. It is not relevant| | | to :py:class:`torch.distributed.FileStore` which does not | | | take in timeout as a parameter. | +--------------+-----------------------------------------------------------+ | is_host | A boolean value indicating whether this backend instance | | | will host the C10d store. If not specified it will be | | | inferred heuristically by matching the hostname or the IP | | | address of this machine against the specified rendezvous | | | endpoint. Defaults to ``None``. | | | | | | Note that this configuration option only applies to | | | :py:class:`torch.distributed.TCPStore`. In normal | | | circumstances you can safely skip it; the only time when | | | it is needed is if its value cannot be correctly | | | determined (e.g. the rendezvous endpoint has a CNAME as | | | the hostname or does not match the FQDN of the machine). | +--------------+-----------------------------------------------------------+ store_typetcpfilez?Invalid store type given. Currently only supports file and tcp.z: )r`rraN)r/striplowerrxrqr"rr Exceptionr typerMrRr FAILED)rVrzrbackendes r&create_backendrsHL%0668>>@J  &v.E 5 %f-EQ (v}}= E> 'Aw''(3q6(3== ''  sAA77 C ACC )(rKloggingrgrsbase64rrdatetimertypingrrrtorch.distributedr r r torch.distributed.elastic.eventsr r apirrrrdynamic_rendezvousrrutilsrr getLoggerrMrjrcrrqrxrTrr,r(r&rs '&&88W 9G   8 $ c#-c#L222x2j3 0;/;E:OQV:V4W;r(