
    %VjigT                        d Z ddlZddlZddlmZ ddlZddlmZ ddgZ G d dej        j	                  Z
 G d d	ej        j	                  Z G d
 dej        j	                  Z G d dej        j	                  ZdS )z
We will recreate all the RNN modules as we require the modules to be decomposed
into its building blocks to be able to observe.
    N)Optional)TensorLSTMCellLSTMc            
           e Zd ZdZej        j        ZdgZ	 	 	 dddde	de	d	e
d
df fdZ	 ddedeeeef                  d
eeef         fdZ	 dde	de
d
eeef         fdZd Zedd            Zedd            Z xZS )r   a  A quantizable long short-term memory (LSTM) cell.

    For the description and the argument types, please, refer to :class:`~torch.nn.LSTMCell`

    `split_gates`: specify True to compute the input/forget/cell/output gates separately
    to avoid an intermediate tensor which is subsequently chunk'd. This optimization can
    be beneficial for on-device inference latency. This flag is cascaded down from the
    parent classes.

    Examples::

        >>> import torch.ao.nn.quantizable as nnqa
        >>> rnn = nnqa.LSTMCell(10, 20)
        >>> input = torch.randn(6, 10)
        >>> hx = torch.randn(3, 20)
        >>> cx = torch.randn(3, 20)
        >>> output = []
        >>> for i in range(6):
        ...     hx, cx = rnn(input[i], (hx, cx))
        ...     output.append(hx)
    split_gatesTNFr   	input_dim
hidden_dimbiasreturnc                   ||d}t                                                       || _        || _        || _        || _        |srt          j        j        |d|z  fd|i|| _	        t          j        j        |d|z  fd|i|| _
        t          j        j        j                                        | _        nt          j                                        | _	        t          j                                        | _
        t          j                                        | _        dD ]v}t          j        j        ||fd|i|| j	        |<   t          j        j        ||fd|i|| j
        |<   t          j        j        j                                        | j        |<   wt          j                                        | _        t          j                                        | _        t          j                                        | _        t          j                                        | _        t          j        j        j                                        | _        t          j        j        j                                        | _        t          j        j        j                                        | _        t          j        j        j                                        | _        d| _        d| _        t          j        | _        t          j        | _        d S )Ndevicedtype   r   )inputforgetcelloutput)      ?r   )super__init__
input_sizehidden_sizer   r   torchnnLinearigateshgatesao	quantizedFloatFunctionalgates
ModuleDictSigmoid
input_gateforget_gateTanh	cell_gateoutput_gatefgate_cxigate_cgatefgate_cx_igate_cgateogate_cyinitial_hidden_state_qparamsinitial_cell_state_qparamsquint8hidden_state_dtypecell_state_dtype)
selfr
   r   r   r   r   r   factory_kwargsg	__class__s
            ]/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/ao/nn/quantizable/modules/rnn.pyr   zLSTMCell.__init__-   s    %+U;;#%	& 	H+08?1z>, ,04,8F, ,DK ,18?A
N, ,15,9G, ,DK +0(+*?*O*O*Q*QDJJ  (--//DK(--//DK,,..DJ: 
H 
H!&z" "04"8F" "A "'
" "15"9G" "A !& 5 E E G G
1(**,, 8++-- 8++---==?? 8;0@@BB$)HK$9$I$I$K$K!-==???G)=E'/4|-2\    xhiddenc                    ||d         |d         &|                      |j        d         |j                  }|\  }}| j        s|                     |          }|                     |          }| j                            ||          }|                    dd          \  }}	}
}| 	                    |          }| 
                    |	          }	|                     |
          }
|                     |          }ni }t          | j                                        | j                                        | j                                                  D ]4\  \  }}}}|                     ||           ||                    ||<   5| 	                    |d                   }| 
                    |d                   }	|                     |d                   }
|                     |d                   }| j                            |	|          }| j                            ||
          }| j                            ||          }|}t)          j        |          }| j                            ||          }||fS )Nr      r   r   r   r   r   )initialize_hiddenshapeis_quantizedr   r   r    r$   addchunkr'   r(   r*   r+   zipitemsvaluesr,   mulr-   r.   r   tanhr/   )r5   r;   r<   hxcxr   r    r$   r'   r(   r*   out_gategatekeyr,   r-   r.   cytanh_cyhys                       r9   forwardzLSTMCell.forwardg   sB    >VAY.&)2C++AGAJGGFB 	8[[^^F[[__FJNN6622E;@;;q!;L;L8JY44J**;77Ky11I''11HH D03
  """"$$""$$1 1 = =,eff
 "IIffQii<<S		g77J**4>::KtF|44I''X77H=$$["55&**:yAA#8<<X{SS! *R..]x112vr:   
batch_sizerA   c                    t          j        || j        f          t          j        || j        f          }}|rN| j        \  }}| j        \  }}t          j        |||| j                  }t          j        |||| j                  }||fS )Nscale
zero_pointr   )r   zerosr   r0   r1   quantize_per_tensorr3   r4   )	r5   rR   rA   hch_scaleh_zpc_scalec_zps	            r9   r?   zLSTMCell.initialize_hidden   s     KT%5677KT%5677   	"?OWd"=OWd)T9P  A )T9N  A !tr:   c                     dS )NQuantizableLSTMCell r5   s    r9   	_get_namezLSTMCell._get_name   s    $$r:   c                    |du |du k    sJ |j         d         }|j         d         } | |||du|          }|st          j                            |          |j        _        |)t          j                            |          |j        _        t          j                            |          |j        _        |)t          j                            |          |j        _        nt          ||g||g|j        |j        g          D ]\  }	}
}t          |		                    dd          |
                                          D ])\  }}t          j                            |          |_        *|
at          |
	                    dd          |
                                          D ])\  }}t          j                            |          |_        *|S )zUses the weights and biases to create a new LSTM cell.

        Args:
            wi, wh: Weights for the input and hidden layers
            bi, bh: Biases for the input and hidden layers
        Nr>   )r
   r   r   r   r   r   )dim)r@   r   r   	Parameterr   weightr   r    rD   rC   rF   )clswiwhbibhr   r   r   r   wbr$   w_chunkrL   b_chunks                  r9   from_paramszLSTMCell.from_params   s    d
d
++++Xa[
hqks "D.#	
 
 
  	@!&!3!3B!7!7DK~#(8#5#5b#9#9 !&!3!3B!7!7DK~#(8#5#5b#9#9   #B8b"XT[7QRR @ @1e%():):ELLNN%K%K > >MGT"'("4"4W"="=DKK=),QWWQAW->->)O)O @ @$)H$6$6w$?$?		r:   c                    t          |          | j        k    sJ t          |d          s
J d            |                     |j        |j        |j        |j        |          }|j        |_        |j        |j	        _        |j        |j
        _        |rP|j	                                        D ]}|j        |_        |j
                                        D ]}|j        |_        |S )Nqconfigz$The float module must have 'qconfig'r	   )type_FLOAT_MODULEhasattrrq   	weight_ih	weight_hhbias_ihbias_hhrs   r   r    rF   )rh   otheruse_precomputed_fake_quantr   observedr7   s         r9   
from_floatzLSTMCell.from_float   s    E{{c/////ui((PP*PPP(??OOMM# # 
 
 !="'-"'- 	*_++-- * *!M		_++-- * *!M		r:   TNNN)F)NNF)FF)__name__
__module____qualname____doc__r   r   r   ru   __constants__intboolr   r   r   tuplerQ   r?   rc   classmethodrq   r~   __classcell__r8   s   @r9   r   r      s        , H%M"OM 8: 8: 8: 8:8: 8: 	8: 
8: 8: 8: 8: 8: 8:v DH) ))!)%*?!@)	vv~	) ) ) )X 5: -1	vv~	   $% % % " " " ["H    [    r:   c            
            e Zd ZdZ	 	 	 ddddededed	df fd
Zddedee	eef                  fdZ
ed             Z xZS )_LSTMSingleLayerzA single one-directional LSTM layer.

    The difference between a layer and a cell is that the layer can process a
    sequence, while the cell only expects an instantaneous value.
    TNFr	   r
   r   r   r   c                ~    ||d}t                                                       t          ||f||d|| _        d S Nr   )r   r   )r   r   r   r   )	r5   r
   r   r   r   r   r   r6   r8   s	           r9   r   z_LSTMSingleLayer.__init__   sY     %+U;;z
(,+
 
IW
 
			r:   r;   r<   c                     g }|j         d         }t          |          D ]9}|                     ||         |          }|                    |d                    :t	          j        |d          }||fS )Nr   )r@   ranger   appendr   stack)r5   r;   r<   resultseq_leniresult_tensors          r9   rQ   z_LSTMSingleLayer.forward   ss    '!*w 	% 	%AYYqtV,,FMM&)$$$$FA..f$$r:   c                 |    t          j        |i |} | |j        |j        |j        |j                  }||_        |S )Nr	   )r   rq   r   r   r   r   r   )rh   argskwargsr   layers        r9   rq   z_LSTMSingleLayer.from_params  sN    #T4V44OT-tydFV
 
 
 
r:   r   r   )r   r   r   r   r   r   r   r   r   r   rQ   r   rq   r   r   s   @r9   r   r      s          
 
 
 

 
 	
 

 
 
 
 
 
 % % %%2G)H % % % %   [    r:   r   c                        e Zd ZdZ	 	 	 	 	 ddddededed	ed
eddf fdZddedee	eef                  fdZ
edd            Z xZS )
_LSTMLayerz#A single bi-directional LSTM layer.TFNr	   r
   r   r   batch_firstbidirectionalr   c                    ||d}	t                                                       || _        || _        t	          ||f||d|	| _        | j        rt	          ||f||d|	| _        d S d S r   )r   r   r   r   r   layer_fwlayer_bw)r5   r
   r   r   r   r   r   r   r   r6   r8   s             r9   r   z_LSTMLayer.__init__  s     %+U;;&*(z
(,+
 
IW
 
  	, '	 
 ! DMMM	 	r:   r;   r<   c                 Z   | j         r|                    dd          }|d\  }}n|\  }}d }| j        r2|d }n|d         }|d         }|d }n|d         }|d         }||||f}||d }n>t          j                            |          t          j                            |          f}|                     ||          \  }	}t          | d          r| j        r	|                    d          }
| 	                    |
|          \  }}|                    d          }t          j
        |	|g|	                                dz
            }||d }d }n|#t          j                            |          \  }}n|#t          j                            |          \  }}nkt          j        |d         |d         gd          }t          j        |d         |d         gd          }n$|	}t          j                            |          \  }}| j         r|                    dd           |||ffS )Nr   r>   )NNr   )r   	transposer   r   jit_unwrap_optionalr   rv   flipr   catre   r   
transpose_)r5   r;   r<   hx_fwcx_fw	hidden_bwhx_bwcx_bw	hidden_fw	result_fw
x_reversed	result_bwr   rY   rZ   s                  r9   rQ   z_LSTMLayer.forward,  sM    	"Aq!!A>'LE55!LE559	 	)}aa}aa U%6!5L	=U]II 	**511	**511I  $}}Q	::	94$$ 	9); 	9J#'==Y#G#G Iy!q))IY	95y}}7JKKF Y%6"33I>>AA"33I>>AAK1y| <a@@K1y| <a@@F9--i88DAq 	$a###1v~r:   r   c                    t          |d          s|J |                    d|j                  }|                    d|j                  }|                    d|j                  }|                    d|j                  }|                    d|j                  }	|                    dd	          }
 | |||||	|

          }t          |d|          |_        t          |d|           }t          |d|           }t          |d| d          }t          |d| d          }t          
                    |||||

          |_        |j        rvt          |d| d          }t          |d| d          }t          |d| dd          }t          |d| dd          }t          
                    |||||

          |_        |S )z
        There is no FP equivalent of this class. This function is here just to
        mimic the behavior of the `prepare` within the `torch.ao.quantization`
        flow.
        rs   Nr   r   r   r   r   r   Fr	   weight_ih_lweight_hh_l	bias_ih_l	bias_hh_l_reverse)rv   getr   r   r   r   r   getattrrs   r   rq   r   r   )rh   r{   	layer_idxrs   r   r   r   r   r   r   r   r   ri   rj   rk   rl   s                   r9   r~   z_LSTMLayer.from_floatc  s    ui((AW-@-@AZZe.>??
jj0ABBzz&%*--jj0ABB

?E4GHHjj66#
 
 
  y'::U5)5566U5)5566U3	33T::U3	33T::)55B 6 
 
  	 Ai A A ABBB Ai A A ABBB ?I ? ? ?FFB ?I ? ? ?FFB-99BBK :  EN r:   )TFFNNr   )r   N)r   r   r   r   r   r   r   r   r   r   rQ   r   r~   r   r   s   @r9   r   r     s        .. !#     	
   
     85 5 5%2G)H 5 5 5 5n ) ) ) [) ) ) ) )r:   r   c                        e Zd ZdZej        j        Z	 	 	 	 	 	 	 dddded	ed
ede	de	de
de	de	ddf fdZddedeeeef                  fdZd Zedd            Zed             Z xZS )r   aX  A quantizable long short-term memory (LSTM).

    For the description and the argument types, please, refer to :class:`~torch.nn.LSTM`

    Attributes:
        layers : instances of the `_LSTMLayer`

    .. note::
        To access the weights and biases, you need to access them per layer.
        See examples below.

    Examples::

        >>> import torch.ao.nn.quantizable as nnqa
        >>> rnn = nnqa.LSTM(10, 20, 2)
        >>> input = torch.randn(5, 3, 10)
        >>> h0 = torch.randn(2, 3, 20)
        >>> c0 = torch.randn(2, 3, 20)
        >>> output, (hn, cn) = rnn(input, (h0, c0))
        >>> # To get the weights:
        >>> # xdoctest: +SKIP
        >>> print(rnn.layers[0].weight_ih)
        tensor([[...]])
        >>> print(rnn.layers[0].weight_hh)
        AssertionError: There is no reverse path in the non-bidirectional layer
    r>   TF        Nr	   r   r   
num_layersr   r   dropoutr   r   r   c
                   
 ||	dt                                                       | _        | _        | _        | _        | _        t          |           _        | _	        d _
        t          |t          j                  r%d|cxk    rdk    rn nt          |t                    rt          d          |dk    r4t!          j        d           |dk    rt!          j        d| d|            t%           j         j         j        fd j	        
d	g}|                     
fd
t)          d|          D                        t*          j                            |           _        d S )Nr   Fr   r>   zbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz|dropout option for quantizable LSTM is ignored. If you are training, please, use nn.LSTM version followed by `prepare` step.zdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=z and num_layers=r   r   r   c              3   f   K   | ]+}t          j        j        j        fd j        dV  ,dS )Fr   N)r   r   r   r   ).0_r6   r5   r   s     r9   	<genexpr>z LSTM.__init__.<locals>.<genexpr>  sv       
 
    	 ""0'  ! 
 
 
 
 
 
r:   )r   r   r   r   r   r   r   floatr   r   training
isinstancenumbersNumberr   
ValueErrorwarningswarnr   extendr   r   r   
ModuleListlayers)r5   r   r   r   r   r   r   r   r   r   r   r   r6   r8   s   `         ` @r9   r   zLSTM.__init__  s    %+U;;$&$	&W~~* 7GN33		$$$$1$$$$$'4(( %   
 Q;;M.  
 Q3BI3 3 '13 3    	 ""0'  ! 

 	 
 
 
 
 
 
 1j))
 
 
 	
 	
 	
 h))&11r:   r;   r<   c                 n   | j         r|                    dd          }|                    d          }| j        rdnd}|t	          j        ||| j        t          j        |j                  	                    d           |j
        rt	          j        dd|j                  fdt          | j                  D             }nt          j                            |          }t#          |d         t$                    rr|d                             | j        ||| j                  |d                             | j        ||| j                  fdt          | j                  D             }n|}g }g }t)          | j                  D ]\  }	}
 |
|||	                   \  }\  }}|                    t          j                            |                     |                    t          j                            |                     t	          j        |          }t	          j        |          }|                    d	|j        d
         |j        d	                   }|                    d	|j        d
         |j        d	                   }| j         r|                    dd          }|||ffS )Nr   r>      )r   r   r   rT   c                     g | ]}fS ra   ra   )r   r   rW   s     r9   
<listcomp>z LSTM.forward.<locals>.<listcomp>  s    CCCqUENCCCr:   c                 |    g | ]8}|                              d           |                              d           f9S )r   )squeeze)r   idxrJ   rI   s     r9   r   z LSTM.forward.<locals>.<listcomp>  sN        W__Q''C););<  r:   )r   r   sizer   r   rW   r   r   r   squeeze_rA   rX   r   r   r   r   r   r   r   reshape	enumerater   r   r   r@   )r5   r;   r<   max_batch_sizenum_directionshxcxhidden_non_opthx_listcx_listr   r   rY   rZ   	hx_tensor	cx_tensorrJ   rI   rW   s                  @@@r9   rQ   zLSTM.forward  s    	"Aq!!A"07a>K kx  E NN1~ 1!'   DCCCE$/,B,BCCCDD"Y77??N.+V44 &#A&..O^^TEU  $A&..O^^TEU     $T_55  
 &#DK00 	: 	:JCac++IAv1NN5955a88999NN5955a889999K((	K((	 %%b)/"*=yr?RSS	%%b)/"*=yr?RSS	 	"Aq!!A9i(((r:   c                     dS )NQuantizableLSTMra   rb   s    r9   rc   zLSTM._get_name/  s      r:   c           
      h   t          || j                  sJ t          |d          s|sJ  | |j        |j        |j        |j        |j        |j        |j	        |          }t          |d|          |_        t          |j                  D ])}t                              |||d|          |j        |<   *|j        r;|                                 t$          j        j                            |d          }n:|                                 t$          j        j                            |d          }|S )Nrs   r	   F)r   r   T)inplace)r   ru   rv   r   r   r   r   r   r   r   r   rs   r   r   r~   r   r   trainr   r!   quantizationprepare_qatevalprepare)rh   r{   rs   r   r}   r   s         r9   r~   zLSTM.from_float2  sA   %!233333ui((3G3333JM#	
 	
 	
 #5)W==)** 	 	C#-#8#8sGK $9 $ $HOC  
 > 	MNNx,8848PPHHMMOOOx,44Xt4LLHr:   c                      t          d          )NzuIt looks like you are trying to convert a non-quantizable LSTM module. Please, see the examples on quantizable LSTMs.)NotImplementedError)rh   r{   s     r9   from_observedzLSTM.from_observedO  s     "1
 
 	
r:   )r>   TFr   FNNr   )NF)r   r   r   r   r   r   r   ru   r   r   r   r   r   r   r   rQ   rc   r   r~   r   r   r   s   @r9   r   r     sh        6 HMM !#H2 "H2 H2 H2H2 H2 	H2
 H2 H2 H2 H2 H2 
H2 H2 H2 H2 H2 H2T5) 5) 5)%2G)H 5) 5) 5) 5)n! ! !    [8 
 
 [
 
 
 
 
r:   )r   r   r   typingr   r   r   __all__r   Moduler   r   r   r   ra   r:   r9   <module>r      s;                   v
M M M M Mux M M M`' ' ' ' 'ux ' ' 'T@ @ @ @ @ @ @ @FG
 G
 G
 G
 G
58? G
 G
 G
 G
 G
r:   