
    2Vh                         d dl mZ d dlmZ d dlmZ  edg       G d dej                               Zej                  j                  dej                        e_        y)	    )ops)keras_export)	optimizerzkeras.optimizers.Lionc                   X     e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z fdZd Z fdZ xZS )Liona  Optimizer that implements the Lion algorithm.

    The Lion optimizer is a stochastic-gradient-descent method that uses the
    sign operator to control the magnitude of the update, unlike other adaptive
    optimizers such as Adam that rely on second-order moments. This makes
    Lion more memory-efficient as it only keeps track of the momentum. According
    to the authors (see reference), its performance gain over Adam grows with
    the batch size. Because the update of Lion is produced through the sign
    operation, resulting in a larger norm, a suitable learning rate for Lion is
    typically 3-10x smaller than that for AdamW. The weight decay for Lion
    should in turn be 3-10x larger than that for AdamW to maintain a
    similar strength (lr * wd).

    Args:
        learning_rate: A float, a
            `keras.optimizers.schedules.LearningRateSchedule` instance, or
            a callable that takes no arguments and returns the actual value to
            use. The learning rate. Defaults to `0.001`.
        beta_1: A float value or a constant float tensor, or a callable
            that takes no arguments and returns the actual value to use. The
            rate to combine the current gradient and the 1st moment estimate.
            Defaults to `0.9`.
        beta_2: A float value or a constant float tensor, or a callable
            that takes no arguments and returns the actual value to use. The
            exponential decay rate for the 1st moment estimate. Defaults to
            `0.99`.
        {{base_optimizer_keyword_args}}

    References:

    - [Chen et al., 2023](http://arxiv.org/abs/2302.06675)
    - [Authors' implementation](
        http://github.com/google/automl/tree/master/lion)

    c                     t        |   d||||||||	|
||d| || _        || _        |dk  s|dkD  rt	        d| d      y )N)learning_ratenameweight_decayclipnorm	clipvalueglobal_clipnormuse_emaema_momentumema_overwrite_frequencyloss_scale_factorgradient_accumulation_stepsr      zqArgument `beta_1` must be in the [0, 1] range. Otherwise, the optimizer degenerates to SignSGD. Received: beta_1=. )super__init__beta_1beta_2
ValueError)selfr	   r   r   r   r   r   r   r   r   r   r   r   r
   kwargs	__class__s                  I/home/dcms/DCMS/lib/python3.12/site-packages/keras/src/optimizers/lion.pyr   zLion.__init__,   s    " 	 	
'%+%$;/(C	
 	
 Q;&1*FFLXQP  %    c                 l    | j                   ryt        | 	  |       | j                  |d      | _        y)zInitialize optimizer variables.

        Lion optimizer has one variable `momentums`.

        Args:
            var_list: list of model variables to build Lion variables on.
        Nmomentum)builtr   buildadd_optimizer_variables
_momentums)r   var_listr   s     r   r$   z
Lion.buildS   s/     ::h66xLr    c                    t        j                  ||j                        }t        j                  ||j                        }t        j                  | j                  |j                        }t        j                  | j                  |j                        }| j
                  | j                  |         }| j                  |t        j                  |t        j                  t        j                  t        j                  ||      t        j                  |d|z
                                 | j                  |t        j                  t        j                  ||      t        j                  |d|z
                     y)z=Update step given gradient and the associated model variable.g      ?N)r   castdtyper   r   r&   _get_variable_index
assign_submultiplysignaddassign)r   gradientvariabler	   lrr   r   ms           r   update_stepzLion.update_step`   s   XXmX^^488Hhnn5$++x~~6$++x~~6OOD44X>?LLGGQ/Xf>	
 	GGQ'hv)O	
r    c                 t    t         |          }|j                  | j                  | j                  d       |S )N)r   r   )r   
get_configupdater   r   )r   configr   s     r   r7   zLion.get_config{   s6    #%++++	
 r    )gMbP?g?Gz?NNNNFr:   NNNlion)	__name__
__module____qualname____doc__r   r$   r5   r7   __classcell__)r   s   @r   r   r      sM    "L  $$(%NM
6 r    r   z{{base_optimizer_keyword_args}}N)	keras.srcr   keras.src.api_exportr   keras.src.optimizersr   	Optimizerr   r?   replacebase_optimizer_keyword_argsr   r    r   <module>rG      s_     - * &'(|9 | )|~ ||##%y'L'Lr    