0;ji'ddlZddlZddlmZddlmcmZddlmZGddejZ GddejZ Gdd ejZ Gd d ejZ dS) N)nn) rearrangec>eZdZd fd Zed dZdZxZS) RelativePositionBiasF ct||_||_||_||_t j|||_dSN) super__init__scalecausal num_buckets max_distancer Embeddingrelative_attention_bias)selfrrrrheads __class__s `/root/voice-cloning/.venv/lib/python3.11/site-packages/chatterbox/models/t3/modules/perceiver.pyr zRelativePositionBias.__init__ sQ   &(')|K'G'G$$$Tc(d}| }|s8|dz}||dk|zz }tj|}n'tj|tj|}|dz}||k}|tj||z tj||z z ||z zz}tj|tj ||dz }|tj |||z }|S)Nr) longtorchabsmax zeros_likelogfloatmathmin full_likewhere) relative_positionrrrretn max_exactis_small val_if_larges r_relative_position_bucketz.RelativePositionBias._relative_position_buckets   2 A K AE<<>>K/ /C ! AA !U-a0011A1$ y=  !''))i/0048L9 () h i j) shaper1rarangerr-rrrrrr) rqk_dotsijr1q_posk_posrel_pos rp_bucketvaluesbiass rforwardzRelativePositionBias.forward+s: bcc*:GN:: 1f Qej@@@ Qej@@@aaa.5D>12274;\`\l@D@Q3SS --i88!455$+,,r)Frrr )Trr)__name__ __module__ __qualname__r staticmethodr-r= __classcell__rs@rrr srHHHHHH\* - - - - - - -rrcJeZdZd fd ZdZd dZd dZd dZd Zd Z xZ S) AttentionQKV皙?NFct||_||_||n|dz|_||_||_tj||_ |r| nd|_ dS)Ng) r r n_headshead_dimrflash dropout_raterDropoutdropoutsetup_flash_config flash_config)rrHrIrKrrJrs rr zAttentionQKV.__init__8s    #/UUX5E  (z,// 9>HD33555Drcdddd}|S)NT) enable_flash enable_mathenable_mem_efficient)rrOs rrNzAttentionQKV.setup_flash_configBs!!$(  rcfd|||fD\}}}jr||||}n||||}|S)Nc:g|]}|SrT) split_heads).0tensorrs r z(AttentionQKV.forward..Ls'DDD4##F++DDDrmask)rJflash_attentionscaled_dot_product_attention combine_heads)rqkvr\outs` rr=zAttentionQKV.forwardKsDDDD1a)DDD1a : H&&q!QT&::CC33Aq!$3GGC!!#&&&rctjd|||jz}|'||dkt d}tj|d}||}tjd||S)Nzbhlt,bhls->bhtsrz-inf)dimzbhts,bhls->bhlt)reinsumr masked_fillr"softmaxrM)rr`rarbr\simattns rr^z)AttentionQKV.scaled_dot_product_attentionTszl,a33dj@  //$!)U6]];;C}Sb)))||D!!|-tQ777rc|jr|jni}tjjjdi|5t j|||||jr|jnd}dddn #1swxYwY|S)Ng) attn_mask dropout_prT) rOrbackendscuda sdp_kernelFr^trainingrK)rr`rarbr\configrcs rr]zAttentionQKV.flash_attention\s&*&7?""R ^ + 5 5f 5 5  01a/3}D$++"C                 s(A  A$'A$c|j\}}}||||j|j}|ddddS)Nrrr)r2viewrHrIpermute)rxbslength_s rrWzAttentionQKV.split_headsfsB FA FF2vt|T] ; ;yyAq!$$$rc|j\}}}}|dddd}|||dS)Nrrrrvre)r2rx contiguousrw)rryrzr|r{s rr_zAttentionQKV.combine_headsksK7Avq IIaAq ! ! , , . .vvb&"%%%r)rFNFr ) r>r?r@r rNr=r^r]rWr_rBrCs@rrErE7sIIIIII''''8888%%% &&&&&&&rrEc8eZdZdZ d fd Zd d ZxZS) AttentionBlock2z An attention block that allows spatial positions to attend to each other, using AttentionQKV and separate linear transformations for Q, K, and V. rreFT皙?Ncdt||_|dkr||_n#||zdksJd|d|||z|_t j||_t j|||_t j|||_ t j|||_ t|j||jz||||_ t j|||_ |r&t||jzdzd|dd |_dSd|_dS) Nrerz channels z' is not divisible by num_head_channels )rKrJrg?Fr@)rrrrr)r r channels num_headsr LayerNormnormLinearto_qto_kto_vrE attentionproj_outrrelative_pos_embeddings) rrrnum_head_channelsrr]rKrrs rr zAttentionBlock2.__init__wsm    " "&DNN,,111_8__L]__211%)::DNL** Ih11 Ih11 Ih11 %dnh$.6P_ktCKPQQQ (H55 " 0+?xSWSaGafhFhqvHVXgi,j,j,jD ( ( (+/D ( ( (rc||j^}}}|j^}}} ||} ||} || } || } || }|| | ||}||}||zj||g|RS)Nr[)r2rrrrrrreshape)rx1x2r\b1c1spatial1b2c2spatial2x1_normx2_normr`rarbhs rr=zAttentionBlock2.forwardsHBHB))B--))B-- IIg   IIg   IIg   NN1aN . . MM!  QB22222r)rreFTrNr r>r?r@__doc__r r=rBrCs@rrrqsp %#0#0#0#0#0#0J33333333rrc*eZdZdZdfd ZdZxZS) Perceiverz,Inspired by https://arxiv.org/abs/2103.03206rcttjtjd|||_tjdtjd||zz z}|jj | |t|||_ dS)a@ Initialize the perceiver module. :param pre_attention_query_token: Number of query tokens for pre-attention :param pre_attention_query_size: Size of each query token :param embedding_dim: Dimension of the embedding space :param num_attn_heads: Number of attention heads rg@g@N) r r rr Parameteremptypre_attention_queryr#sqrtdatauniform_rrk)rpre_attention_query_tokenpre_attention_query_size embedding_dimnum_attn_headsquery_variancers rr zPerceiver.__init__s $)8#5#5 K46N O O$ $  3$)C;TWp;p4q*r*rr  %..OOO$M>BB rc|j|jddd}|||}|||}|S)z Forward pass of the perceiver module. :param h: Input tensor :return: Output after applying attention mechanisms rre)rexpandr2rk)rrquery_pre_attrks rr=zPerceiver.forwardsO)00RDD))FA&&yy'** r)rrrrrrCs@rrrsX66CCCCCC2       rr) r#rrtorch.nn.functional functionalrreinopsrModulerrErrrTrrrs (-(-(-(-(-29(-(-(-V7&7&7&7&7&297&7&7&t9393939393bi939393x''''' '''''r