VhyUddlmZddlZddlmcmZddlmZddlm Z gZ e e e d<ejjGddZy))OptionalN)Tensor)2_scripted_functional_optimizer_deprecation_warning__all__ceZdZ ddeededeeefdededededed ed efd Zd ed e efdZ dee efdZ y)_FunctionalAdamWparamslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc ftdd|kstd|d|kstd|d|dcxkrdksntd|dd|d cxkrdksntd |d d|kstd ||||d|d |d |_||_||_||_| |_tjjttjtttjffi|_ t|dk(r | s td d|i|_y)N) stacklevelzInvalid learning rate: zInvalid epsilon value: rg?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )r r beta1beta2r z%optimizer got an empty parameter listr )r ValueErrordefaultsrrrrtorchjitannotatedictrstrstatelen param_group) selfr r r r r rrrrrs X/home/dcms/DCMS/lib/python3.12/site-packages/torch/distributed/optim/functional_adamw.py__init__z_FunctionalAdamW.__init__sJ ;aHby6rd;< <cz6se<= =eAh$$B58*MN NeAh$$B58*MN Nl";L>JK K1X1X(        YY''U\\4U\\@Q;R-R(SUWX v;! $;DE E%f-paramgradcfg}g}g}g}g}g}tj|} |"|j||j|||jvri|j|<|j|} tjd| d<tj |tj | d<tj |tj | d<|jr(tj |tj | d<|j|} |j| d|j| d|jr|j| d|j| dtj5tj|||||||j|j|jd|jd|jd |jd |jd |j|jdd| dddy#1swYyxYw) Nrstep memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrr r r rrrrr r r rr grad_scale found_inf has_complex)r is_complexappendr!tensor zeros_likepreserve_formatrno_gradFadamwrrrr) r$r(r)params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr4r!s r% step_paramz_FunctionalAdamW.step_paramGs $& &&u-    # #E * LL   " "DJJu JJu%E!LL-E&M$//U%:%: E) #("2"2U%:%:#E, ||*/*:*:)>)>+&' 5!i()5./ <<  " "5)9#: ;5=) ]]_  GG  mmG,mmG,==&!]]>:MM%( jj'%    s BH''H0 gradientscX|jd}g}g}g}g}g}g}t|t|k7r*tddt|dzdt|zd} t|jd|D]v\} } |  | t j | z} |j | |j | | |jvri|j| <|j| } t jd| d<t j| tj | d <t j| tj | d <|jr(t j| tj | d <|j| } |j | d |j | d |jr|j | d |j | dyt j5tj|||||||j|j|j d |j d|j d|j d|j d|j"|j$dd| dddy#1swYyxYw)Nr zEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: Frr+r,r.r/r0rrr r r r1)r#r"rziprr5r6r!r7r8r9rr:r;r<rrrr) r$rDr r=r>r?r@rArBr4r(gradientr!s r%r+z_FunctionalAdamW.steps!!(+ $& v;#i. (W#CK=34&s9~&678   "4#3#3H#=yI 2OE8#u//66  ''. X& *(*DJJu% JJu-E$)LL$5E&M','7'7U-B-B(E)$+0*:*:U-B-B+E,'||272B2B!1F1F3./ 5)i 01""5#67<<#**51A+BC""5=1A 2D]]_  GG  mmG,mmG,==&!]]>:MM%( jj'%    s BJ  J)N) gMbP?)g?g+?g:0yE>g{Gz?FFFFF) __name__ __module__ __qualname__listrfloattupleboolr&rrCr+r'r%rrs %1"(-+.V +. +.UE\" +.  +.  +.+.+.+.+."&+.Z;;hv.>;zGd8F#34Gr'r)typingrrtorch.optim._functionaloptim _functionalr;r,torch.distributed.optim._deprecation_warningrrrKr __annotations__rscriptrrOr'r%rWsO ## crrrr'