VhUddlmZddlZddlmcmZddlmZddlm Z gZ e e e d<ejjGddZy))OptionalN)Tensor)2_scripted_functional_optimizer_deprecation_warning__all__ceZdZ ddeededeeefdededededed ed efd Zd ed e efdZ dee efdZ y)_FunctionalAdamparamslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc ftdd|kstd|d|kstd|d|dcxkrdksntd|dd|d cxkrdksntd |d d|kstd ||||d|d |d |_||_||_||_| |_tjjttjtttjffi|_ t|dk(r | s td d|i|_y)N) stacklevelzInvalid learning rate: zInvalid epsilon value: rg?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )r r beta1beta2r z%optimizer got an empty parameter listr )r ValueErrordefaultsrrrrtorchjitannotatedictrstrstatelen param_group) selfr r r r r rrrrrs W/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/optim/functional_adam.py__init__z_FunctionalAdam.__init__sJ ;aHby6rd;< <cz6se<= =eAh$$B58*MN NeAh$$B58*MN Nl";L>JK K1X1X(        YY''U\\4U\\@Q;R-R(SUWX v;! $;DE E%f-paramgradcfg}g}g}g}g}g}tj|} |"|j||j|||jvri|j|<|j|} tjd| d<tj |tj | d<tj |tj | d<|jr(tj |tj | d<|j|} |j| d|j| d|jr|j| d|j| dtj5tj|||||||j| |j|jd|jd |jd |jd |jd |j|jdd dddy#1swYyxYw)zo Similar to step, but operates on a single parameter and optionally a gradient tensor. Nrstep memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrr r r r has_complexrrrr r r rr grad_scale found_inf)r is_complexappendr!tensor zeros_likepreserve_formatrno_gradFadamrrrr) r$r(r)params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr2r!s r% step_paramz_FunctionalAdam.step_paramGs  $& &&u-    # #E * LL   " "DJJu JJu%E!LL-E&M$//U%:%: E) #("2"2U%:%:#E, ||*/*:*:)>)>+&' 5!i()5./ <<  " "5)9#: ;5=) ]]_  FF  'mmG,mmG,==&!]]>:MM%( jj%    s BH''H0 gradientscX|jd}g}g}g}g}g}g}d} t|t|k7r*tddt|dzdt|zt|jd|D]v\} } |  | t j | z} |j | |j | | |jvri|j| <|j| } t jd| d<t j| tj | d <t j| tj | d <|jr(t j| tj | d <|j| } |j | d |j | d |jr|j | d |j | dyt j5tj|||||||j| |j|j d |j d|j d|j d|j d|j"|j$dddddy#1swYyxYw)Nr FzEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: rr+r,r.r/r0rrr r r r1)r#r"rziprr5r6r!r7r8r9rr:r;r<rrrr) r$rDr r=r>r?r@rArBr2r(gradientr!s r%r+z_FunctionalAdam.steps!!(+ $&  v;#i. (W#CK=34&s9~&678  #4#3#3H#=yI 2OE8#u//66  ''. X& *(*DJJu% JJu-E$)LL$5E&M','7'7U-B-B(E)$+0*:*:U-B-B+E,'||272B2B!1F1F3./ 5)i 01""5#67<<#**51A+BC""5=1A 2D]]_  FF  'mmG,mmG,==&!]]>:MM%( jj%    s BJ  J)N) gMbP?)g?g+?g:0yE>rFFFFF) __name__ __module__ __qualname__listrfloattupleboolr&rrCr+r'r%rrs %1!(-+.V +. +.UE\" +.  +.  +.+.+.+.+."&+.Z::hv.>:xGd8F#34Gr'r)typingrrtorch.optim._functionaloptim _functionalr;r,torch.distributed.optim._deprecation_warningrrrKr __annotations__rscriptrrOr'r%rWsO ## cqqqr'