
    VhCT                     8   d Z ddlZddlZddlmZ ddlZddlmZ ddgZ G d dej                  j                        Z
 G d d	ej                  j                        Z G d
 dej                  j                        Z G d dej                  j                        Zy)z
We will recreate all the RNN modules as we require the modules to be decomposed
into its building blocks to be able to observe.
    N)Optional)TensorLSTMCellLSTMc            
            e Zd ZdZej
                  j                  ZdgZ	 	 	 dddde	de	de
d	df fd
Z	 ddedeeeef      d	eeef   fdZ	 dde	de
d	eeef   fdZd Zedd       Zedd       Z xZS )r   a  A quantizable long short-term memory (LSTM) cell.

    For the description and the argument types, please, refer to :class:`~torch.nn.LSTMCell`

    `split_gates`: specify True to compute the input/forget/cell/output gates separately
    to avoid an intermediate tensor which is subsequently chunk'd. This optimization can
    be beneficial for on-device inference latency. This flag is cascaded down from the
    parent classes.

    Examples::

        >>> import torch.ao.nn.quantizable as nnqa
        >>> rnn = nnqa.LSTMCell(10, 20)
        >>> input = torch.randn(6, 10)
        >>> hx = torch.randn(3, 20)
        >>> cx = torch.randn(3, 20)
        >>> output = []
        >>> for i in range(6):
        ...     hx, cx = rnn(input[i], (hx, cx))
        ...     output.append(hx)
    split_gatesNFr   	input_dim
hidden_dimbiasreturnc                   ||d}t         	|           || _        || _        || _        || _        |st        j                  j                  |d|z  fd|i|| _	        t        j                  j                  |d|z  fd|i|| _
        t        j                  j                  j                  j                         | _        nt        j                  j                         | _	        t        j                  j                         | _
        t        j                  j                         | _        dD ]  }t        j                  j                  ||fd|i|| j                  |<   t        j                  j                  ||fd|i|| j                  |<   t        j                  j                  j                  j                         | j                  |<    t        j                  j!                         | _        t        j                  j!                         | _        t        j                  j'                         | _        t        j                  j!                         | _        t        j                  j                  j                  j                         | _        t        j                  j                  j                  j                         | _        t        j                  j                  j                  j                         | _        t        j                  j                  j                  j                         | _        d| _        d| _        t        j8                  | _        t        j8                  | _        y )Ndevicedtype   r   )inputforgetcelloutput)      ?r   )super__init__
input_sizehidden_sizer   r   torchnnLinearigateshgatesao	quantizedFloatFunctionalgates
ModuleDictSigmoid
input_gateforget_gateTanh	cell_gateoutput_gatefgate_cxigate_cgatefgate_cx_igate_cgateogate_cyinitial_hidden_state_qparamsinitial_cell_state_qparamsquint8hidden_state_dtypecell_state_dtype)
selfr
   r   r   r   r   r   factory_kwargsg	__class__s
            S/home/dcms/DCMS/lib/python3.12/site-packages/torch/ao/nn/quantizable/modules/rnn.pyr   zLSTMCell.__init__,   s    %+U;#%	&+088??1z>,04,8F,DK ,188??A
N,15,9G,DK +0((++*?*?*O*O*QDJ  ((--/DK((--/DK,,.DJ: 
H!&z"04"8F"A "'
"15"9G"A !& 5 5 E E G

1
H  ((**, 88++- 88++---==? 88;;00@@B$)HHKK$9$9$I$I$K!--==??G)=E'/4||-2\\    xhiddenc                    |
|d   |d   )| j                  |j                  d   |j                        }|\  }}| j                  s| j	                  |      }| j                  |      }| j                  j                  ||      }|j                  dd      \  }}	}
}| j                  |      }| j                  |	      }	| j                  |
      }
| j                  |      }ni }t        | j                  j                         | j                  j                         | j
                  j                               D ]*  \  \  }}}}|j                   ||       ||            ||<   , | j                  |d         }| j                  |d         }	| j                  |d         }
| j                  |d         }| j                   j#                  |	|      }| j$                  j#                  ||
      }| j&                  j                  ||      }|}t)        j*                  |      }| j,                  j#                  ||      }||fS )Nr      r   r   r   r   r   )initialize_hiddenshapeis_quantizedr   r   r    r$   addchunkr'   r(   r*   r+   zipitemsvaluesr,   mulr-   r.   r   tanhr/   )r5   r;   r<   hxcxr   r    r$   r'   r(   r*   out_gategatekeyr,   r-   r.   cytanh_cyhys                       r9   forwardzLSTMCell.forwardf   s
    >VAY.&)2C++AGGAJGFB[[^F[[_FJJNN662E;@;;q!;L8JY4J**;7Ky1I''1H D03

  """$""$1 =,eff
 "IIfQi<S	= g7J**4>:KtF|4I''X7H==$$["5&&**:yA#88<<X{S! **R.]]x12vr:   
batch_sizerA   c                 Z   t        j                  || j                  f      t        j                  || j                  f      }}|rd| j                  \  }}| j                  \  }}t        j
                  |||| j                        }t        j
                  |||| j                        }||fS )Nscale
zero_pointr   )r   zerosr   r0   r1   quantize_per_tensorr3   r4   )	r5   rR   rA   hch_scaleh_zpc_scalec_zps	            r9   r?   zLSTMCell.initialize_hidden   s     {{J(8(89:EKK))*=
1 "??OWd"==OWd))T9P9PA ))T9N9NA !tr:   c                      y)NQuantizableLSTMCell r5   s    r9   	_get_namezLSTMCell._get_name   s    $r:   c                    |du |du k(  sJ |j                   d   }|j                   d   } | |||du|      }|st        j                  j                  |      |j                  _        |.t        j                  j                  |      |j                  _        t        j                  j                  |      |j                  _        |.t        j                  j                  |      |j                  _        |S t        ||g||g|j                  |j                  g      D ]  \  }	}
}t        |	j                  dd      |j                               D ])  \  }}t        j                  j                  |      |_        + |
at        |
j                  dd      |j                               D ])  \  }}t        j                  j                  |      |_        +  |S )zUses the weights and biases to create a new LSTM cell.

        Args:
            wi, wh: Weights for the input and hidden layers
            bi, bh: Biases for the input and hidden layers
        Nr>   )r
   r   r   r   r   r   )dim)r@   r   r   	Parameterr   weightr   r    rD   rC   rF   )clswiwhbibhr   r   r   r   wbr$   w_chunkrL   b_chunks                  r9   from_paramszLSTMCell.from_params   s    d
d
+++XXa[
hhqk "D.#	
 !&!3!3B!7DKK~#(88#5#5b#9 !&!3!3B!7DKK~#(88#5#5b#9    #B8b"XT[[7QR @1e%():ELLN%K >MGT"'(("4"4W"=DK> =),QWWQAW->)O @$)HH$6$6w$?	@@ r:   c                 .   t        |      | j                  k(  sJ t        |d      sJ d       | j                  |j                  |j
                  |j                  |j                  |      }|j                  |_        |j                  |j                  _        |j                  |j                  _        |r`|j                  j                         D ]  }|j                  |_         |j                  j                         D ]  }|j                  |_         |S )Nqconfigz$The float module must have 'qconfig'r	   )type_FLOAT_MODULEhasattrrq   	weight_ih	weight_hhbias_ihbias_hhrs   r   r    rF   )rh   otheruse_precomputed_fake_quantr   observedr7   s         r9   
from_floatzLSTMCell.from_float   s    E{c/////ui(P*PP(??OOOOMMMM# # 
 !=="'--"'--__++- *!MM	*__++- *!MM	*r:   TNNN)F)NNF)FF)__name__
__module____qualname____doc__r   r   r   ru   __constants__intboolr   r   r   tuplerQ   r?   rc   classmethodrq   r~   __classcell__r8   s   @r9   r   r      s    * HH%%M"OM 8: 8:8: 8: 	8: 
8:v DH))!)%*?!@)	vv~	)X 5:-1	vv~	"% " "H  r:   c            
       t     e Zd ZdZ	 	 	 ddddedededdf fd	Zdd
edee	eef      fdZ
ed        Z xZS )_LSTMSingleLayerzA single one-directional LSTM layer.

    The difference between a layer and a cell is that the layer can process a
    sequence, while the cell only expects an instantaneous value.
    NFr	   r
   r   r   r   c                V    ||d}t         |           t        ||f||d|| _        y Nr   )r   r   )r   r   r   r   )	r5   r
   r   r   r   r   r   r6   r8   s	           r9   r   z_LSTMSingleLayer.__init__   s=     %+U;z
(,+
IW
	r:   r;   r<   c                     g }|j                   d   }t        |      D ]+  }| j                  ||   |      }|j                  |d          - t	        j
                  |d      }||fS )Nr   )r@   ranger   appendr   stack)r5   r;   r<   resultseq_leniresult_tensors          r9   rQ   z_LSTMSingleLayer.forward   sh    ''!*w 	%AYYqtV,FMM&)$	% FA.f$$r:   c                     t        j                  |i |} | |j                  |j                  |j                  |j
                        }||_        |S )Nr	   )r   rq   r   r   r   r   r   )rh   argskwargsr   layers        r9   rq   z_LSTMSingleLayer.from_params  sL    ##T4V4OOT--tyydFVFV
 
r:   r   r   )r   r   r   r   r   r   r   r   r   r   rQ   r   rq   r   r   s   @r9   r   r      s}     
 

 
 	
 

 % %%2G)H %  r:   r   c                        e Zd ZdZ	 	 	 	 	 ddddedededed	ed
df fdZddedee	eef      fdZ
edd       Z xZS )
_LSTMLayerz#A single bi-directional LSTM layer.FNr	   r
   r   r   batch_firstbidirectionalr   c                    ||d}	t         
|           || _        || _        t	        ||f||d|	| _        | j                  rt	        ||f||d|	| _        y y r   )r   r   r   r   r   layer_fwlayer_bw)r5   r
   r   r   r   r   r   r   r   r6   r8   s             r9   r   z_LSTMLayer.__init__  s     %+U;&*(z
(,+
IW
 , '	
 !DM r:   r;   r<   c                    | j                   r|j                  dd      }|d\  }}n|\  }}d }| j                  r&|d }n
|d   }|d   }|d }n
|d   }|d   }||||f}||d }n>t        j                  j                  |      t        j                  j                  |      f}| j                  ||      \  }	}t        | d      r| j                  r|j                  d      }
| j                  |
|      \  }}|j                  d      }t        j                  |	|g|	j                         dz
        }||d }d }n|#t        j                  j                  |      \  }}n|#t        j                  j                  |      \  }}nat        j                  |d   |d   gd      }t        j                  |d   |d   gd      }n$|	}t        j                  j                  |      \  }}| j                   r|j                  dd       |||ffS )Nr   r>   )NNr   )r   	transposer   r   jit_unwrap_optionalr   rv   flipr   catre   r   
transpose_)r5   r;   r<   hx_fwcx_fw	hidden_bwhx_bwcx_bw	hidden_fw	result_fw
x_reversed	result_bwr   rY   rZ   s                  r9   rQ   z_LSTMLayer.forward*  s   Aq!A>'LE5!LE559	}aa}aa U%6!5L	=U]I		2259599;U;U< I  $}}Q	:	94$););J#'==Y#G Iy!q)IYY	95y}}7JKF Y%6"33I>A"33I>AKK1y| <a@KK1y| <a@F99--i8DAqa#1v~r:   c                 6   t        |d      s|J |j                  d|j                        }|j                  d|j                        }|j                  d|j                        }|j                  d|j
                        }|j                  d|j                        }	|j                  dd	      }
 | |||||	|

      }t        |d|      |_        t        |d|       }t        |d|       }t        |d| d      }t        |d| d      }t        j                  |||||

      |_        |j                  rat        |d| d      }t        |d| d      }t        |d| dd      }t        |d| dd      }t        j                  |||||

      |_        |S )z
        There is no FP equivalent of this class. This function is here just to
        mimic the behavior of the `prepare` within the `torch.ao.quantization`
        flow.
        rs   Nr   r   r   r   r   r   Fr	   weight_ih_lweight_hh_l	bias_ih_l	bias_hh_l_reverse)rv   getr   r   r   r   r   getattrrs   r   rq   r   r   )rh   r{   	layer_idxrs   r   r   r   r   r   r   r   r   ri   rj   rk   rl   s                   r9   r~   z_LSTMLayer.from_float`  s    ui(W-@AAZZe.>.>?
jj0A0ABzz&%**-jj0A0AB

?E4G4GHjj6#
  y':Uk)56Uk)56Ui	{3T:Ui	{3T:)55B 6 
 +i[ ABB+i[ ABB)I;h ?FB)I;h ?FB-99BBK : EN r:   )TFFNNr   )r   N)r   r   r   r   r   r   r   r   r   r   rQ   r   r~   r   r   s   @r9   r   r     s    . !#   	
   
84 4%2G)H 4l ) )r:   r   c                        e Zd ZdZej
                  j                  Z	 	 	 	 	 	 	 ddddededede	d	e	d
e
de	de	ddf fdZddedeeeef      fdZd Zedd       Zed        Z xZS )r   aX  A quantizable long short-term memory (LSTM).

    For the description and the argument types, please, refer to :class:`~torch.nn.LSTM`

    Attributes:
        layers : instances of the `_LSTMLayer`

    .. note::
        To access the weights and biases, you need to access them per layer.
        See examples below.

    Examples::

        >>> import torch.ao.nn.quantizable as nnqa
        >>> rnn = nnqa.LSTM(10, 20, 2)
        >>> input = torch.randn(5, 3, 10)
        >>> h0 = torch.randn(2, 3, 20)
        >>> c0 = torch.randn(2, 3, 20)
        >>> output, (hn, cn) = rnn(input, (h0, c0))
        >>> # To get the weights:
        >>> # xdoctest: +SKIP
        >>> print(rnn.layers[0].weight_ih)
        tensor([[...]])
        >>> print(rnn.layers[0].weight_hh)
        AssertionError: There is no reverse path in the non-bidirectional layer
    FNr	   r   r   
num_layersr   r   dropoutr   r   r   c
                   
 ||	dt                    | _        | _        | _        | _        | _        t        |       _        | _	        d _
        t        |t        j                        r(d|cxk  rdk  rn t        d      t        |t              rt        d      |dkD  r5t!        j"                  d       |dk(  rt!        j"                  d| d|        t%         j                   j                   j
                  fd j                  
d	g}|j'                   
fd
t)        d|      D               t*        j,                  j/                  |       _        y )Nr   Fr   r>   zbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz|dropout option for quantizable LSTM is ignored. If you are training, please, use nn.LSTM version followed by `prepare` step.zdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=z and num_layers=r   r   r   c              3      K   | ]>  }t        j                  j                  j                  fd j                  d @ yw)Fr   N)r   r   r   r   ).0_r6   r5   r   s     r9   	<genexpr>z LSTM.__init__.<locals>.<genexpr>  sY      
      		 ""00' !
s   AA)r   r   r   r   r   r   r   floatr   r   training
isinstancenumbersNumberr   
ValueErrorwarningswarnr   extendr   r   r   
ModuleListlayers)r5   r   r   r   r   r   r   r   r   r   r   r   r6   r8   s   `         ` @r9   r   zLSTM.__init__  sv    %+U;$&$	&W~* 7GNN3$1$   '4( 
 Q;MM.
 QBBI K&&0\3   		 ""00' !

 	 
 1j)
 	
 hh))&1r:   r;   r<   c                    | j                   r|j                  dd      }|j                  d      }| j                  rdnd}|t	        j
                  ||| j                  t        j                  |j                        }|j                  d       |j                  r#t	        j                  |dd|j                        }t        | j                        D cg c]  }||f }}nt        j                  j!                  |      }t#        |d   t$              r|d   j'                  | j                  ||| j                        }	|d   j'                  | j                  ||| j                        }
t        | j                        D cg c]*  }|	|   j)                  d      |
|   j)                  d      f, }}n|}g }g }t+        | j,                        D ]s  \  }} ||||         \  }\  }}|j/                  t        j                  j!                  |             |j/                  t        j                  j!                  |             u t	        j0                  |      }t	        j0                  |      }|j'                  d|j2                  d   |j2                  d         }|j'                  d|j2                  d   |j2                  d         }| j                   r|j                  dd      }|||ffS c c}w c c}w )	Nr   r>      )r   r   r   rT   )r   r   sizer   r   rW   r   r   r   squeeze_rA   rX   r   r   r   r   r   r   r   reshapesqueeze	enumerater   r   r   r@   )r5   r;   r<   max_batch_sizenum_directionsrW   r   hxcxhidden_non_optrI   rJ   idxhx_listcx_listr   rY   rZ   	hx_tensor	cx_tensors                      r9   rQ   zLSTM.forward  s   Aq!A"00a>KK  kkxxE NN1~~11!'' -2$//,BCqUENCDC"YY77?N.+V4#A&..OO^^TEUEU $A&..OO^^TEUEU
  %T__5 W__Q'C);< 
 &#DKK0 	:JCac+IAv1NN59955a89NN59955a89	: KK(	KK(	 %%b)//"*=yr?RS	%%b)//"*=yr?RS	Aq!A9i(((E Ds   K-/K2c                      y)NQuantizableLSTMra   rb   s    r9   rc   zLSTM._get_name+  s     r:   c           
         t        || j                        sJ t        |d      s|sJ  | |j                  |j                  |j
                  |j                  |j                  |j                  |j                  |      }t        |d|      |_        t        |j
                        D ])  }t        j                  |||d|      |j                  |<   + |j                   r=|j#                          t$        j&                  j(                  j+                  |d      }|S |j-                          t$        j&                  j(                  j/                  |d      }|S )Nrs   r	   F)r   r   T)inplace)r   ru   rv   r   r   r   r   r   r   r   r   rs   r   r   r~   r   r   trainr   r!   quantizationprepare_qatevalprepare)rh   r{   rs   r   r}   r   s         r9   r~   zLSTM.from_float.  s0   %!2!2333ui(G33JJMM#	
 #5)W=))* 	C#-#8#8sGK $9 $HOOC 	 >>NNxx,,8848PH  MMOxx,,44Xt4LHr:   c                     t        d      )NzuIt looks like you are trying to convert a non-quantizable LSTM module. Please, see the examples on quantizable LSTMs.)NotImplementedError)rh   r{   s     r9   from_observedzLSTM.from_observedK  s     "1
 	
r:   )r>   TFg        FNNr   )NF)r   r   r   r   r   r   r   ru   r   r   r   r   r   r   r   rQ   rc   r   r~   r   r   r   s   @r9   r   r     s    4 HHMMM !#H2 "H2H2 H2 	H2
 H2 H2 H2 H2 H2 
H2T5) 5)%2G)H 5)n!  8 
 
r:   )r   r   r   typingr   r   r   __all__r   Moduler   r   r   r   ra   r:   r9   <module>r      s         v
Kuxx K\'uxx 'T DF
588?? F
r:   