
    /;ji&                        d dl Z d dlmZmZmZ d dlZd dlmZ d dlmc m	Z
 ddlmZ ddlmZ ddlmZmZ  G d d	ej                  Z G d
 dej                  Z G d dej                  Z G d dej                  Z G d dej                  Z edd          rej        Zn G d dej                  Z G d dej                  Z G d dej                  ZdS )    N)DictOptionalTuple   )is_torch_version   )get_activation)CombinedTimestepLabelEmbeddings)PixArtAlphaCombinedTimestepSizeEmbeddingsc                   ^     e Zd ZdZdedef fdZdej        dej        dej        fdZ xZ	S )	AdaLayerNormz
    Norm layer modified to incorporate timestep embeddings.

    Parameters:
        embedding_dim (`int`): The size of each embedding vector.
        num_embeddings (`int`): The size of the embeddings dictionary.
    embedding_dimnum_embeddingsc                    t                                                       t          j        ||          | _        t          j                    | _        t          j        ||dz            | _        t          j	        |d          | _
        d S )Nr   F)elementwise_affine)super__init__nn	EmbeddingembSiLUsiluLinearlinear	LayerNormnormselfr   r   	__class__s      X/root/voice-cloning/.venv/lib/python3.11/site-packages/diffusers/models/normalization.pyr   zAdaLayerNorm.__init__%   sk    <>>GII	i}q/@AAL5III			    xtimestepreturnc                     |                      |                     |                     |                              }t          j        |d          \  }}|                     |          d|z   z  |z   }|S )Nr   r   )r   r   r   torchchunkr   )r   r"   r#   r   scaleshifts         r    forwardzAdaLayerNorm.forward,   sb    kk$))DHHX$6$67788{3**uIIaLLAI&.r!   )
__name__
__module____qualname____doc__intr   r&   Tensorr*   __classcell__r   s   @r    r   r      s         Jc J3 J J J J J J  %,        r!   r   c                       e Zd ZdZddedee         f fdZ	 	 	 	 ddej        deej                 deej	                 d	eej
                 d
eej                 deej        ej        ej        ej        ej        f         fdZ xZS )AdaLayerNormZeroz
    Norm layer adaptive layer norm zero (adaLN-Zero).

    Parameters:
        embedding_dim (`int`): The size of each embedding vector.
        num_embeddings (`int`): The size of the embeddings dictionary.
    Nr   r   c                 ,   t                                                       |t          ||          | _        nd | _        t	          j                    | _        t	          j        |d|z  d          | _        t	          j	        |dd          | _
        d S )N   TbiasFư>)r   eps)r   r   r
   r   r   r   r   r   r   r   r   r   s      r    r   zAdaLayerNormZero.__init__<   s    %6~}UUDHHDHGII	iq=/@tLLLL5dSSS			r!   r"   r#   class_labelshidden_dtyper   r$   c                 >   | j         |                      |||          }|                     |                     |                    }|                    dd          \  }}}}	}
}|                     |          d|d d d f         z   z  |d d d f         z   }|||	|
|fS )N)r<   r6   r   dim)r   r   r   r'   r   )r   r"   r#   r;   r<   r   	shift_msa	scale_msagate_msa	shift_mlp	scale_mlpgate_mlps               r    r*   zAdaLayerNormZero.forwardG   s     8((8\(MMCkk$))C..))ILSTZ[I\I\F	9h	9hIIaLLA	!!!T' 223i46HH(Iy(::r!   N)NNNN)r+   r,   r-   r.   r/   r   r   r&   r0   
LongTensordtyper   r*   r1   r2   s   @r    r4   r4   3   s        	T 	Tc 	T8C= 	T 	T 	T 	T 	T 	T ,037.2&*; ;<; 5<(; u/0	;
 u{+; el#; 
u|U\5<u|S	T; ; ; ; ; ; ; ;r!   r4   c                        e Zd ZdZddedef fdZ	 	 	 ddej        de	e
eej        f                  d	e	e         d
e	ej                 deej        ej        ej        ej        ej        f         f
dZ xZS )AdaLayerNormSingleaL  
    Norm layer adaptive layer norm single (adaLN-single).

    As proposed in PixArt-Alpha (see: https://arxiv.org/abs/2310.00426; Section 2.3).

    Parameters:
        embedding_dim (`int`): The size of each embedding vector.
        use_additional_conditions (`bool`): To use additional conditions for normalization or not.
    Fr   use_additional_conditionsc                     t                                                       t          ||dz  |          | _        t	          j                    | _        t	          j        |d|z  d          | _        d S )N   )size_emb_dimrK   r6   Tr7   )	r   r   r   r   r   r   r   r   r   )r   r   rK   r   s      r    r   zAdaLayerNormSingle.__init__b   sl    <(:Vo
 
 
 GII	iq=/@tLLLr!   Nr#   added_cond_kwargs
batch_sizer<   r$   c                 |     | j         |fi |||d}|                     |                     |                    |fS )N)rP   r<   )r   r   r   )r   r#   rO   rP   r<   embedded_timesteps         r    r*   zAdaLayerNormSingle.forwardl   sR     %DHXuu1Buzhtuuuu{{499%67788:KKKr!   )F)NNN)r+   r,   r-   r.   r/   boolr   r&   r0   r   r   strrH   r   r*   r1   r2   s   @r    rJ   rJ   W   s         M Mc Md M M M M M M @D$(.2	L 	L,	L $Del):$;<	L SM		L
 u{+	L 
u|U\5<u|S	T	L 	L 	L 	L 	L 	L 	L 	Lr!   rJ   c                   z     e Zd ZdZ	 ddedededee         def
 fd	Zd
e	j
        de	j
        de	j
        fdZ xZS )AdaGroupNorma  
    GroupNorm layer modified to incorporate timestep embeddings.

    Parameters:
        embedding_dim (`int`): The size of each embedding vector.
        num_embeddings (`int`): The size of the embeddings dictionary.
        num_groups (`int`): The number of groups to separate the channels into.
        act_fn (`str`, *optional*, defaults to `None`): The activation function to use.
        eps (`float`, *optional*, defaults to `1e-5`): The epsilon value to use for numerical stability.
    Nh㈵>r   out_dim
num_groupsact_fnr:   c                     t                                                       || _        || _        |d | _        nt          |          | _        t          j        ||dz            | _        d S )Nr   )	r   r   rY   r:   actr	   r   r   r   )r   r   rX   rY   rZ   r:   r   s         r    r   zAdaGroupNorm.__init__   sa     	$>DHH%f--DHiw{;;r!   r"   r   r$   c                    | j         r|                      |          }|                     |          }|d d d d d d f         }|                    dd          \  }}t          j        || j        | j                  }|d|z   z  |z   }|S )Nr   r   r>   )r:   )r\   r   r'   F
group_normrY   r:   )r   r"   r   r(   r)   s        r    r*   zAdaGroupNorm.forward   s    8 	 ((3--Ckk#!!!QQQd"#yyy**uLDO:::UOe#r!   )NrW   )r+   r,   r-   r.   r/   r   rT   floatr   r&   r0   r*   r1   r2   s   @r    rV   rV   x   s        	 	 mq< < <+.<<?<IQRU<di< < < < < <	 	EL 	U\ 	 	 	 	 	 	 	 	r!   rV   c                   d     e Zd Z	 	 	 	 ddedef fdZdej        dej        d	ej        fd
Z xZS )AdaLayerNormContinuousTrW   
layer_normr   conditioning_embedding_dimc                 P   t                                                       t          j                    | _        t          j        ||dz  |          | _        |dk    rt          ||||          | _        d S |dk    rt          |||          | _        d S t          d|           )Nr   r7   rc   rms_normzunknown norm_type )r   r   r   r   r   r   r   r   r   RMSNorm
ValueError)r   r   rd   r   r:   r8   	norm_typer   s          r    r   zAdaLayerNormContinuous.__init__   s     	GII	i :MA<MTXYYY$$!-6H$OODIII*$$s4FGGDIII=)==>>>r!   r"   conditioning_embeddingr$   c                 *   |                      |                     |                              |j                            }t	          j        |dd          \  }}|                     |          d|z   d d d d d f         z  |d d d d d f         z   }|S )Nr   r   r>   )r   r   torH   r&   r'   r   )r   r"   rj   r   r(   r)   s         r    r*   zAdaLayerNormContinuous.forward   s    kk$))$:;;>>qwGGHH{3q111uIIaLLAIqqq$z22U111dAAA:5FFr!   )TrW   Trc   )	r+   r,   r-   r/   r   r&   r0   r*   r1   r2   s   @r    rb   rb      s          ? ?? %(? ? ? ? ? ?0 u| PUP\        r!   rb   z>=z2.1.0c                   4     e Zd Zddededef fdZd Z xZS )	r   rW   Tr:   r   r8   c                    t                                                       || _        t          |t          j                  r|f}t          j        |          | _        |r\t          j
        t          j        |                    | _        |r&t          j
        t          j        |                    nd | _        d S d | _        d | _        d S rF   )r   r   r:   
isinstancenumbersIntegralr&   Sizer?   r   	Parameteronesweightzerosr8   )r   r?   r:   r   r8   r   s        r    r   zLayerNorm.__init__   s    GGDH#w/00 fz#DH! ! l5:c??;;>BLBLS)9)9:::			" 			r!   c                 Z    t          j        || j        | j        | j        | j                  S rF   )r^   rc   r?   ru   r8   r:   )r   inputs     r    r*   zLayerNorm.forward   s"    <txdiRRRr!   )rW   TTr+   r,   r-   r`   rS   r   r*   r1   r2   s   @r    r   r      sn        	! 	!U 	!t 	!Z^ 	! 	! 	! 	! 	! 	!"	S 	S 	S 	S 	S 	S 	Sr!   r   c                   0     e Zd Zddedef fdZd Z xZS )rg   Tr:   r   c                 .   t                                                       || _        t          |t          j                  r|f}t          j        |          | _        |r-t          j
        t          j        |                    | _        d S d | _        d S rF   )r   r   r:   ro   rp   rq   r&   rr   r?   r   rs   rt   ru   )r   r?   r:   r   r   s       r    r   zRMSNorm.__init__   s{    c7+,, 	&C:c?? 	,uz#77DKKKDKKKr!   c                    |j         }|                    t          j                                      d                              dd          }|t          j        || j        z             z  }| j        N| j        j         t          j	        t          j
        fv r|                    | j        j                   }|| j        z  }n|                    |          }|S )Nr   T)keepdim)rH   rl   r&   float32powmeanrsqrtr:   ru   float16bfloat16)r   hidden_statesinput_dtypevariances       r    r*   zRMSNorm.forward   s    #) ##EM2266q99>>r4>PP%Htx4G(H(HH;"{ U]EN$CCC - 0 01B C C)DK7MM),,[99Mr!   )Try   r2   s   @r    rg   rg      s_          D            r!   rg   c                   $     e Zd Z fdZd Z xZS )GlobalResponseNormc                     t                                                       t          j        t	          j        ddd|                    | _        t          j        t	          j        ddd|                    | _        d S )Nr   )r   r   r   rs   r&   rv   gammabeta)r   r?   r   s     r    r   zGlobalResponseNorm.__init__   s_    \%+aAs";";<<
LQ1c!:!:;;			r!   c                     t          j        |ddd          }||                    dd          dz   z  }| j        ||z  z  | j        z   |z   S )Nr   )r   r   T)pr?   r~   r}   )r?   r~   r9   )r&   r   r   r   r   )r   r"   gxnxs       r    r*   zGlobalResponseNorm.forward  sV    ZQFD999277r4700478zQV$ty0144r!   )r+   r,   r-   r   r*   r1   r2   s   @r    r   r      sG        < < < < <
5 5 5 5 5 5 5r!   r   )rp   typingr   r   r   r&   torch.nnr   torch.nn.functional
functionalr^   utilsr   activationsr	   
embeddingsr
   r   Moduler   r4   rJ   rV   rb   r   rg   r    r!   r    <module>r      s@     ( ( ( ( ( ( ( ( ( (                 $ $ $ $ $ $ ' ' ' ' ' ' b b b b b b b b    29   .!; !; !; !; !;ry !; !; !;HL L L L L L L LB# # # # #29 # # #L    RY   B D'"" SIIS S S S SBI S S S,    bi   @
5 
5 
5 
5 
5 
5 
5 
5 
5 
5r!   