
    2Vh|                         d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ  ed       G d de             Zd Zd Zd Zd Zy)    N)backend)constraints)initializers)ops)regularizers)keras_export)is_flash_attention_enabled)Softmax)EinsumDense)Layer)Dropoutzkeras.layers.MultiHeadAttentionc                   r    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Zed        Zed        Zed        Zed        Z	ed        Z
ed        Z fd	Z	 dd
Zed        Zed        Zed        Zed        Zd ZddZd ZddZ	 	 	 ddZ	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 ddZddZ	 ddZ	 	 	 	 	 	 	 	 ddZ xZS )MultiHeadAttentiona  MultiHeadAttention layer.

    This is an implementation of multi-headed attention as described in the
    paper "Attention is all you Need"
    [Vaswani et al., 2017](https://arxiv.org/abs/1706.03762).
    If `query`, `key,` `value` are the same, then
    this is self-attention. Each timestep in `query` attends to the
    corresponding sequence in `key`, and returns a fixed-width vector.

    This layer first projects `query`, `key` and `value`. These are
    (effectively) a list of tensors of length `num_attention_heads`, where the
    corresponding shapes are `(batch_size, <query dimensions>, key_dim)`,
    `(batch_size, <key/value dimensions>, key_dim)`,
    `(batch_size, <key/value dimensions>, value_dim)`.

    Then, the query and key tensors are dot-producted and scaled. These are
    softmaxed to obtain attention probabilities. The value tensors are then
    interpolated by these probabilities, then concatenated back to a single
    tensor.

    Finally, the result tensor with the last dimension as `value_dim` can take
    a linear projection and return.

    Args:
        num_heads: Number of attention heads.
        key_dim: Size of each attention head for query and key.
        value_dim: Size of each attention head for value.
        dropout: Dropout probability.
        use_bias: Boolean, whether the dense layers use bias vectors/matrices.
        output_shape: The expected shape of an output tensor, besides the batch
            and sequence dims. If not specified, projects back to the query
            feature dim (the query input's last dimension).
        attention_axes: axes over which the attention is applied. `None` means
            attention over all axes, but batch, heads, and features.
        flash_attention: If `None`, the layer attempts to use flash
            attention for faster and more memory-efficient attention
            computations when possible. This behavior can be configured using
            `keras.config.enable_flash_attention()` or
            `keras.config.disable_flash_attention()`.
        kernel_initializer: Initializer for dense layer kernels.
        bias_initializer: Initializer for dense layer biases.
        kernel_regularizer: Regularizer for dense layer kernels.
        bias_regularizer: Regularizer for dense layer biases.
        activity_regularizer: Regularizer for dense layer activity.
        kernel_constraint: Constraint for dense layer kernels.
        bias_constraint: Constraint for dense layer kernels.
        seed: Optional integer to seed the dropout layer.

    Call arguments:
        query: Query tensor of shape `(B, T, dim)`, where `B` is the batch size,
            `T` is the target sequence length, and dim is the feature dimension.
        value: Value tensor of shape `(B, S, dim)`, where `B` is the batch size,
            `S` is the source sequence length, and dim is the feature dimension.
        key: Optional key tensor of shape `(B, S, dim)`. If not given, will
            use `value` for both `key` and `value`, which is the most common
            case.
        attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
            attention to certain positions. The boolean mask specifies which
            query elements can attend to which key elements, 1 indicates
            attention and 0 indicates no attention. Broadcasting can happen for
            the missing batch dimensions and the head dimension.
        return_attention_scores: A boolean to indicate whether the output should
            be `(attention_output, attention_scores)` if `True`, or
            `attention_output` if `False`. Defaults to `False`.
        training: Python boolean indicating whether the layer should behave in
            training mode (adding dropout) or in inference mode (no dropout).
            Will go with either using the training mode of the parent
            layer/model, or `False` (inference) if there is no parent layer.
        use_causal_mask: A boolean to indicate whether to apply a causal mask to
            prevent tokens from attending to future tokens (e.g., used in a
            decoder Transformer).

    Returns:
        attention_output: The result of the computation, of shape `(B, T, E)`,
            where `T` is for target sequence shapes and `E` is the query input
            last dimension if `output_shape` is `None`. Otherwise, the
            multi-head outputs are projected to the shape specified by
            `output_shape`.
        attention_scores: (Optional) multi-head attention coefficients over
            attention axes.
    c                    t        |   di | d| _        || _        || _        |r|n|| _        || _        || _        |rt        |t              r|f}	 t        |      }|| _        |xs
 t               | _        t        j                   |	      | _        t        j                   |
      | _        t'        j                   |      | _        t'        j                   |      | _        t'        j                   |      | _        t/        j                   |      | _        t/        j                   |      | _        t        |t              r|f}n&|r$t        |t4        t        f      st        d|       || _        || _        dt;        j<                  t?        | j                              z  | _         | j                  r| j                  dkD  rt        d      y y #  t        d| d      xY w)	NTzInvalid `output_shape`: zK. When specified, the `output_shape` should be of type tuple, list, or int.zI`attention_axes` must be an int, list, or tuple.Received: attention_axes=g      ?        zkDropout is not supported when flash attention is enabled. Please set dropout to 0.0 to use flash attention. )!super__init__supports_masking
_num_heads_key_dim
_value_dim_dropout	_use_bias
isinstanceinttuple
ValueError_output_shaper	   _flash_attentionr   get_kernel_initializer_bias_initializerr   _kernel_regularizer_bias_regularizer_activity_regularizerr   _kernel_constraint_bias_constraintlist_attention_axesseedmathsqrtfloat_inverse_sqrt_key_dim)self	num_headskey_dim	value_dimdropoutuse_biasoutput_shapeattention_axesflash_attentionkernel_initializerbias_initializerkernel_regularizerbias_regularizeractivity_regularizerkernel_constraintbias_constraintr+   kwargs	__class__s                     _/home/dcms/DCMS/lib/python3.12/site-packages/keras/src/layers/attention/multi_head_attention.pyr   zMultiHeadAttention.__init__g   s   ( 	"6" $#'0)g!,, ,$\2 * / O3M3O#/#3#34F#G !-!1!12B!C#/#3#34F#G !-!1!12B!C%1%5%56J%K""-//2C"D + @nc*,.NJ~e}$M,,:+;=   .	%(499U4==5I+J%J"   T]]S%8D  &9 9 .|n =$ $ s   G G!c                     | j                   S N)r   r0   s    rB   r1   zMultiHeadAttention.num_heads           c                     | j                   S rD   )r   rE   s    rB   r2   zMultiHeadAttention.key_dim       }}rG   c                     | j                   S rD   )r   rE   s    rB   r3   zMultiHeadAttention.value_dim   rF   rG   c                     | j                   S rD   )r   rE   s    rB   r4   zMultiHeadAttention.dropout   rI   rG   c                     | j                   S rD   )r   rE   s    rB   r5   zMultiHeadAttention.use_bias   s    ~~rG   c                     | j                   S rD   )r*   rE   s    rB   r7   z!MultiHeadAttention.attention_axes   s    ###rG   c                    t         |          }| j                  | j                  | j                  | j
                  | j                  | j                  | j                  t        j                  | j                        t        j                  | j                        t        j                  | j                        t        j                  | j                        t        j                  | j                         t#        j                  | j$                        t#        j                  | j&                        | j(                  d}i ||S )N)r1   r2   r3   r4   r5   r6   r7   r9   r:   r;   r<   r=   r>   r?   r+   )r   
get_configr   r   r   r   r   r   r*   r   	serializer"   r#   r   r$   r%   r&   r   r'   r(   r+   )r0   base_configconfigrA   s      rB   rO   zMultiHeadAttention.get_config   s    g(*}}}} .."22"."8"8((# !- 6 6t7M7M N"."8"8((# !- 6 6t7M7M N$0$:$:**% "-!6!6t7N7N!O*44T5J5JKII+
. )+(((rG   c                    ||n|}|dd |dd k7  rt        d| d|       t        |      }t        |      }t        |      }t        |dz
  dd      \  }}}	t        |ft	        |	dz
  | j
                  | j                  g      | j                  r|nddd	| j                         | _	        | j                  j                  |       t        |dz
  dd      \  }}}	t        |ft	        |	dz
  | j
                  | j                  g      | j                  r|ndd
d	| j                         | _        | j                  j                  |       t        |dz
  dd      \  }}}	t        |ft	        |	dz
  | j
                  | j                  g      | j                  r|nddd	| j                         | _        | j                  j                  |       | j                  |	       | j                  || j                         d      | _        t#        | j                  j%                  |            }
| j                  |
d<   | j                   j                  t'        |
             y)zBuilds layers and variables.

        Args:
            query_shape: Shape of the `query` tensor.
            value_shape: Shape of the `value` tensor.
            key: Optional shape of the `key` tensor.
        N   _All dimensions of `value` and `key`, except the last one, must be equal. Received: value_shape= and key_shape=   
bound_dimsoutput_dimsqueryr6   	bias_axesnamekeyvalueattention_output)r   len_build_proj_equationr   _get_output_shaper   r   r   _get_common_kwargs_for_sublayer_query_densebuild
_key_denser   _value_dense_build_attention_make_output_dense_output_denser)   compute_output_shaper   )r0   query_shapevalue_shape	key_shape
query_rank
value_rankkey_rankeinsum_equationr^   output_rankoutput_dense_input_shapes              rB   rh   zMultiHeadAttention.build   s    $-#4K)	q	!B/88C} E&K)  %
%
y>2FNqa3
/K (
*a$//4==!A $(>>it
 224
 	,2FqLQA3
/K &
*a$//4==!A $(>>it
 224
 	i(2FNqa3
/K (
*a$//4??!C $(>>it
 224
 	,
 	k*!44002

 $(22;?$
  (, $  '?!@ArG   c                     | j                   S rD   )rg   rE   s    rB   query_densezMultiHeadAttention.query_dense/         rG   c                     | j                   S rD   )ri   rE   s    rB   	key_densezMultiHeadAttention.key_dense3  rF   rG   c                     | j                   S rD   )rj   rE   s    rB   value_densezMultiHeadAttention.value_dense7  rz   rG   c                     | j                   S rD   )rm   rE   s    rB   output_densezMultiHeadAttention.output_dense;  s    !!!rG   c                    t        | j                  | j                  | j                  | j                  | j
                  | j                        }| j                  j                  j                  | j                  j                               }| j                  j                  j                  | j                  j                               }||d<   ||d<   |S )N)r;   r<   r=   r>   r?   dtyper9   r:   )dictr$   r%   r&   r'   r(   dtype_policyr"   rA   from_configrO   r#   )r0   common_kwargsr9   r:   s       rB   rf   z2MultiHeadAttention._get_common_kwargs_for_sublayer?  s    #77!33!%!;!;"55 11##
 "55??KK$$//1
  11;;GG""--/
 /A*+,<()rG   c                     t        |      }| j                  r| j                  }n|d   g}t        |dz
  dt        |            \  }}}t        |ft	        |dz
  |      | j
                  r|nd|d|S )a2  Builds the output projection matrix.

        Args:
            free_dims: Number of free dimensions for einsum equation building.
            common_kwargs: Common keyword arguments for einsum layer.
            name: Name for the projection layer.

        Returns:
            Projection layer.
        rU   rT   rX   rY   Nr]   )rc   r   rd   r   re   r   )	r0   ro   r   r_   rr   r6   ru   r^   rv   s	            rB   rl   z%MultiHeadAttention._make_output_denseU  s     %
--L'O,L2FNqc,6G3
/K 
*;?LI#'>>it	

 
 	
rG   c                    | j                   t        t        d|dz
              | _         nt        | j                         | _         t        || j                         \  | _        | _        }t        t        |t        | j                         z
  |            }t        || j                        | _	        t        | j                  | j                  | j                        | _        y)a3  Builds multi-head dot-product attention computations.

        This function builds attributes necessary for `_compute_attention` to
        customize attention computation to replace the default dot-product
        attention.

        Args:
            rank: the rank of query, key, value tensors.
        NrT   rX   )	attn_axes)axisr   )rater   r+   )r*   r   range_build_attention_equation_dot_product_equation_combine_equationrc   r
   r   _softmaxr   r   r+   _dropout_layer)r0   rankattn_scores_rank	norm_axess       rB   rk   z#MultiHeadAttention._build_attentionp  s     '#(q$();#<D #()=)=#>D 
 &dd6J6JK		
&" 3t';';#<<>N
	
  Yd6G6GH%d&7&7dii
rG   c                    |lt        | j                         dz  dz
  }t        t        |j                        t        |j                        z
        D ]  }t	        j
                  ||      } | j                  ||      S )NrX   rT   r   )mask)rc   r*   r   shaper   expand_dimsr   )r0   attention_scoresattention_maskmask_expansion_axis_s        rB   _masked_softmaxz"MultiHeadAttention._masked_softmax  s     % $'t';';#<"<q"@1"D$**+c.2F2F.GG  "%")<" }}-N}CCrG   c           
      \   | j                   r|rt        d      | j                  dkD  xs |xs t        |j                        dk7   }|r|rt        | j
                         dz  dz
  }d}	t        |	t        |j                        z
        D ]  }
t        j                  ||      } t        j                  |d	      }t        j                  |||d|| j                  d
| j                         }|dfS t        j                  |t        j                  | j                  |j                              }t        j                  | j                  ||      }| j!                  ||      }| j                  dkD  r| j#                  ||      }n|}t        j                  | j$                  ||      }||fS )a   Applies Dot-product attention with query, key, value tensors.

        This function defines the computation inside `call` with projected
        multi-head Q, K, V inputs. Users can override this function for
        customized attention implementation.

        Args:
            query: Projected query tensor of shape `(B, T, N, key_dim)`.
            key: Projected key tensor of shape `(B, S, N, key_dim)`.
            value: Projected value tensor of shape `(B, S, N, value_dim)`.
            attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
                attention to certain positions. It is generally not needed if
                the `query` and `value` (and/or `key`) are masked.
            training: Python boolean indicating whether the layer should behave
                in training mode (adding dropout) or in inference mode (doing
                nothing).

        Returns:
          attention_output: Multi-headed outputs of attention computation.
          attention_scores: Multi-headed attention weights.
        zReturning attention scores is not supported when flash attention is enabled. Please disable flash attention to access attention scores.r      NrX   rT   r   boolr   F)r\   r`   ra   biasr   scale	is_causalr8   )training)r    r   r   rc   r   r*   r   r   r   castdot_product_attentionr/   multiplyr   einsumr   r   r   r   )r0   r\   r`   ra   r   r   return_attention_scoresuse_dot_product_attentionr   len_attention_scores_shaper   rb   r   final_attn_scoress                 rB   _compute_attentionz%MultiHeadAttention._compute_attention  s   >   %<%  MMC '&'EKK A%%
! %) (+4+?+?'@&@1&Dq&H#-.*.^5I5I1JJ A &)__&-@&N "%.!G"88#00 $ 5 5	  $T)) 388D66D
 ::d&@&@#uM  //n

 ==3 $ 3 3 8 !4 ! !1 ::""$5u
  !111rG   c           	         ||}t        j                  |      }t        j                  |d        t        j                  |d        t        j                  |d        | j                  |||||||
      }| j	                  |      }| j                  |      }| j                  |      }| j                  |||||	|      \  }}| j                  |      }|t        j                  ||       |r||fS |S )N)
query_mask
value_maskkey_maskr   use_causal_mask)	r   get_keras_maskset_keras_mask_compute_attention_maskrg   ri   rj   r   rm   )r0   r\   ra   r`   r   r   r   r   r   r   r   rb   r   s                rB   callzMultiHeadAttention.call  s    ;C ++E2
ud+ud+sD)55!!)+ 6 
 !!%( ooc" !!%(-1-D-D#.
**  --.>? !""#3Z@"#%555rG   c                    d}|,t        j                  |d      }t        j                  |d      }|5t        j                  |d      }t        j                  |d      }	||	n||	z  }|5t        j                  |d      }t        j                  |d      }	||	n||	z  }|r| j                  ||      }	||	n||	z  }|t        j                  |d      }|	||n||z  }|S )a  Computes the attention mask, using the Keras masks of the inputs.

        * The `query`'s mask is reshaped from [B, T] to [B, T, 1].
        * The `value`'s mask is reshaped from [B, S] to [B, 1, S].
        * The `key`'s mask is reshaped from [B, S] to [B, 1, S]. The `key`'s
          mask is ignored if `key` is `None` or if `key is value`.
        * If `use_causal_mask=True`, then the causal mask is computed. Its shape
          is [1, T, S].

        All defined masks are merged using a logical AND operation (`&`).

        In general, if the `query` and `value` are masked, then there is no need
        to define the `attention_mask`.

        Args:
            query: Projected query tensor of shape `(B, T, N, key_dim)`.
            key: Projected key tensor of shape `(B, T, N, key_dim)`.
            value: Projected value tensor of shape `(B, T, N, value_dim)`.
            attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
                attention to certain positions.
            use_causal_mask: A boolean to indicate whether to apply a causal
                mask to prevent tokens from attending to future tokens (e.g.,
                used in a decoder Transformer).

        Returns:
            attention_mask: a boolean mask of shape `(B, T, S)`, that prevents
                attention to certain positions, based on the Keras masks of the
                `query`, `key`, `value`, and `attention_mask` tensors, and the
                causal mask if `use_causal_mask=True`.
        Nr   rU   )r   r   r   _compute_causal_mask)
r0   r\   ra   r   r   r   r   r   	auto_maskr   s
             rB   r   z*MultiHeadAttention._compute_attention_mask?  s   P 	!*f5J
B7I!*f5J??:r2D ) 1y47GIxx&1H??8R0D ) 1y47GI,,UE:D ) 1y47GI% XXnf=N  ") #i/ 
 rG   c                 &   t        j                  |      d   }||nt        j                  |      d   }t        j                  d||fd      }t        j                  |d      }t        j                  |d      }t        j                  ||      S )a  Computes a causal mask (e.g., for masked self-attention layers).

        For example, if query and value both contain sequences of length 4,
        this function returns a boolean tensor equal to:

        ```
        [[[True,  False, False, False],
          [True,  True,  False, False],
          [True,  True,  True,  False],
          [True,  True,  True,  True]]]
        ```

        Args:
            query: query tensor of shape `(B, T, ...)`.
            value: value tensor of shape `(B, S, ...)` (optional, defaults to
                query).

        Returns:
            mask: a boolean tensor of shape `(1, T, S)` containing a lower
                triangular matrix of shape `(T, S)`.
        rT   int32r   r   r   rU   )r   r   onescumsumgreater_equal)r0   r\   ra   q_seq_lengthv_seq_length	ones_mask	row_index	col_indexs           rB   r   z'MultiHeadAttention._compute_causal_mask  sz    , yy'*',}|#))E:J1:MHHa|<GL	JJyr2	JJyr2	  I66rG   c                     t        |      }t        |      }||}nt        |      }|dd |dd k7  rt        d| d|       | j                  r|d d | j                  z   }|S )NrT   rU   rV   rW   )r   r   r   )r0   ro   rp   rq   s       rB   rn   z'MultiHeadAttention.compute_output_shape  s     K(K(#Ii(Iq	!B/88C} E&K) 
 %cr*T-?-??KrG   c                 Z   ||j                   }nd }| j                  |j                   |j                   |      }t        j                  || j                        }|rO|j                   d   }|j                   d   | j
                  ||f}|t        j                  || j                        fS |S )Nr   rT   r   )r   rn   r   KerasTensorcompute_dtyper1   )r0   r\   ra   r`   r   r   r   r   r   r   r   rq   r6   output_speclengthattention_shapes                   rB   compute_output_specz&MultiHeadAttention.compute_output_spec  s     ?		II00KKi
 )) 2 2
 #[[^F${{1~t~~vvNO 3 3t'9'9!   rG   )Nr   TNNNglorot_uniformzerosNNNNNNrD   )NNF)NNNNNFNF)NNNNF)__name__
__module____qualname____doc__r   propertyr1   r2   r3   r4   r5   r7   rO   rh   ry   r|   r~   r   rf   rl   rk   r   r   r   r   r   rn   r   __classcell__)rA   s   @rB   r   r      s   Pl + !#@D           $ $)> 	NB` ! !   ! ! " ",
6
:D*  %c2R  %; B EN7B 	6  %rG   r   c                 (    t         j                  |    S )z~Converts an index to a einsum variable name.

    We simply map indices to lowercase characters, e.g. 0 -> 'a', 1 -> 'b'.
    )stringascii_lowercase)is    rB   _index_to_einsum_variabler     s    
 !!!$$rG   c                    d}t        |       D ]  }|t        |      z  } t        t        j                  t        |       || dz
  fz               }| }d}t        |       D ]*  }||v s|| dz
  k(  r	|||   z  }|t        |      z  }|dz  }, dj                  |D cg c]  }||   	 c}|D cg c]  }||   	 c}z   |D cg c]  }||   	 c}z         }|d|d|}t        |      }	|d|d|}
||
|	fS c c}w c c}w c c}w )a  Builds einsum equations for the attention computation.

    Query, key, value inputs after projection are expected to have the shape as:
    `(bs, <non-attention dims>, <attention dims>, num_heads, channels)`.
    `bs` and `<non-attention dims>` are treated as `<batch dims>`.

    The attention operations can be generalized:
    1. Query-key dot product:
        (<batch dims>, <query attention dims>, num_heads, channels),
        (<batch dims>, <key attention dims>, num_heads, channels) ->
        (<batch dims>, num_heads, <query attention dims>, <key attention dims>)
    2. Combination:
        (<batch dims>, num_heads, <query attention dims>, <key attention dims>),
        (<batch dims>, <value attention dims>, num_heads, channels) -> (<batch
        dims>, <query attention dims>, num_heads, channels)

    Args:
        rank: Rank of query, key, value tensors.
        attn_axes: List/tuple of axes, `[-1, rank)`,
            that attention will be applied to.

    Returns:
        Einsum equations.
     rT   ,->)r   r   r   npdeletejoinrc   )r   r   target_notationr   
batch_dimsletter_offsetsource_notationproduct_notationdot_product_equationr   combine_equations              rB   r   r     sK   2 O4[ 84Q778 ryyti4!8+.EFGJMO4[ 
?a4!8mq11O8GGOQM ww%/0	0'0
1!?1
1	2'0
1!?1
1	2 	
 +,
  !13CCC 	1
1
1s   C:-C?
D
c                 T   d}d}d}d}d}t        |       D ]  }t        ||z         }	||	z  }||	z  } || z  }t        |      D ]  }t        ||z         }	||	z  }||	z  } ||z  }t        |      D ]  }t        ||z         }	||	z  }||	z  }||	z  }! | d| d| }
|
|t        |      fS )zFBuilds an einsum equation for projections inside multi-head attention.r   r   r   r   )r   r   rc   )	free_dimsrZ   r[   	input_str
kernel_str
output_strr^   r   r   charequations              rB   rd   rd     s   IJJIM9 (]):;T	d

 YM: (]):;T	d

 ZM; (]):;d
d
T		
 Aj\J<8HYJ//rG   c                 >    d g| t        |      z
  z  t        |      z   S rD   )rc   r)   )rv   known_last_dimss     rB   re   re   9  s#    6[3#7784;PPPrG   )r,   r   numpyr   	keras.srcr   r   r   r   r   keras.src.api_exportr   keras.src.backend.configr	   $keras.src.layers.activations.softmaxr
   "keras.src.layers.core.einsum_denser   keras.src.layers.layerr   'keras.src.layers.regularization.dropoutr   r   r   r   rd   re   r   rG   rB   <module>r      sk        ! "  " - ? 8 : ( ; /0C C 1CL%7Dt0:QrG   