QVji@#UddlZddlmZmZmZddlmZddlZddlm Z edZ edZ iZ e ejjefed<ehdZd ed ed eee e fgee e fffd Zed d d4dddddejdejdejdeejdededed ejfdZdeeded efdZdejdeded ejfdZdejd ejd!ed"edeed#ed ejfd$Zd!ed"ed dfd%Zdejd ejd!ed"edeed ejf d&Zed'd d5dddddd(dd)dejd ejd*ejd+eejd,eejd-eejd.ed/ed0ed#edeed1ed2eed eejejejejffd3Z dS)6N)CallableOptionalTypeVar) ParamSpec)_dtype_mappings_P_RONNX_ATEN_DECOMP_TABLE> op_type opset_versionreturnc|dtttfdtttfffd }|S)zDDecorator to register an ONNX operator with a custom implementation.funcrcd}tjdd|d|}|tt t tjj|<|||S)Nopsetzonnx::.) mutates_args)torchlibrary custom_opr getattropsonnx register_fake)roverloadtorch_oprrs N/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/onnx/ops/_impl.py decoratorz_onnx_op..decorators*=**=** )W ) )x ) )+    wwuy~w'G'GRRS t$$$)rrr )rrr#s`` r"_onnx_opr%sO R( Xb"f-=        r$RotaryEmbeddingF) interleaved num_headsrotary_embedding_dimx cos_cache sin_cache position_idsr(r)r*ch|jt}ddtjdkfdtjjdkfdtjjdkfdtjdkodkfd nGtjd kod kfd |d krtj|d }nJ|d krDtj|dkfdd}||z} || g} tj|| }tjt|jd kd|jd } |dkr| }|ddddddd|f} |dddddd|df} |dzntjjdkojdkfdtjjdkojdkfdtjjdkfdtjjdkfdtjdtjd|r+| dddddddddf} | dddddddddf}ntj| dd\} }| z|zz }| z|zz}|r]tj|d}tj|d}tj ||fd}tj|| j} ntj ||fd} tj | | fd}|d krtj|Stj|d S)z_RotaryEmbedding-23 https://onnx.ai/onnx/operators/onnx__RotaryEmbedding.html#rotaryembedding-23rNcdjS)Nz6position_ids must be 2D when provided. Received shape shape)r.sr"z%rotary_embedding_23..Fsa\M_aar$c*ddjdS)Nz6position_ids first dim (batch) must match x.shape[0] ( ). Received rr3) batch_sizer.sr"r5z%rotary_embedding_23..Js"|Z||eqewxyez||r$r c*ddjdS)Nz;position_ids second dim (sequence) must match x.shape[-2] (r7r r3)r.sequence_lengthsr"r5z%rotary_embedding_23..Ns>GRaGGo{pBCDpEGGr$c(djdjS)NzWcos_cache/sin_cache must be 2D when position_ids is provided. Received cos_cache shape , sin_cache shape r3r,r-sr"r5z%rotary_embedding_23..R-](1]]KT?]]r$c(djdjS)Nz[cos_cache/sin_cache must be 3D when position_ids is not provided. Received cos_cache shape r<r3r=sr"r5z%rotary_embedding_23..Xr>r$)rr1r r?cdS)NzKnum_heads must be provided for 3D inputs. Received input tensor with shape r) input_shapesr"r5z%rotary_embedding_23..esobmoor$cdS)Nzx should be a 4D tensor by nowrrr$r"r5z%rotary_embedding_23..ls,Lr$c&djdddS)Nzcos has shape  but expected (batch=, seq=, ...)r3)r8cosr:sr"r5z%rotary_embedding_23..&jjjjjSbjjjr$c&djdddS)Nzsin has shape rFrGrHr3)r8r:sinsr"r5z%rotary_embedding_23..rJr$c,djdddS)NzLast dimension of cos cache (rM') should match rotary_embedding_dim/2 ().r3)rIrotary_embedding_dim_halfsr"r5z%rotary_embedding_23..4D " DDfDDDr$c,djdddS)NzLast dimension of sin cache (rMrOrPr3)rQrLsr"r5z%rotary_embedding_23..rRr$dim) r4lenr_checkrUpermutereshape unsqueezechunkcat)r+r,r-r.r(r)r* input_rank hidden_size head_size new_shapex_rotate x_not_rotatex1x2realimagx_rotate_concatoutputr8rIrCrQr:rLs ``` @@@@@@r"rotary_embedding_23ri/sv'K[!!JQJ!"oO     ! # a a a a      q !Z / | | | | |      q !_ 4 G G G G G     MMOOq 9Y]]__%9 ] ] ] ] ]     MMOOq 9Y]]__%9 ] ] ] ] ]   Q M!\ * * q N o o o o   "!n 9, )YG M!Y ' ' LQW"$L$LMMM Iq  (AAAqqq////0HQQQ11123334L 4 9     L ! "Fsy|'Fjjjjjj L ! "Fsy|'Fjjjjjj L " 22 D D D D D L " 22 D D D D D / Q  C / Q  C 2 aaaAAAqt!tm $ aaaAAAqt!tm $Xqb111B 8cBh D 8cBh D3tR((tR(()T4Lb999=(.AA9dD\r222 Y,/R 8 8 8FQ}V[111 = . ..r$scaler_c8||ndtj|z S)z/Get the scale factor for attention computation.Ng?)mathsqrt)rjr_s r"_get_scale_factorrns %55C$)I2F2F,FGr$tensorr8c|jd|jd}}||z}|||||ddS)z1Reshape 3D tensor to 4D for multi-head attention.r r1)r4view transpose contiguous)ror8r)r:r^r_s r"_reshape_3d_to_4drtsS$*<?FLO[Oy(I JIFF 1a r$QKcurrent_q_num_headscurrent_kv_num_headsqk_matmul_output_modec |dkrt|||||Stjtj||ddS)z1Get QK output tensor based on the specified mode.rr0rM)_compute_qk_output_for_mode_0r zeros_likematmulrr)rurvrwrxrjrys r"_get_qk_output_for_aten_spdar~s[!!, q%';U    Q B0C0C D DEEEr$cJtjzdkfddS)z-Validate Group Query Attention configuration.rcdddS)Nz q_num_heads (z%) must be divisible by kv_num_heads (z ) for GQAr)rxrwsr"r5z-_validate_gqa_configuration..sy 3yyZnyyyr$N)rrW)rwrxs``r"_validate_gqa_configurationrsA L22a7yyyyyr$c|}||kr||z}||d}t||jd}tj|}||z} ||z} t j| | ddS)zDHelper function to compute QK output for qk_matmul_output_mode == 0.r rTr?r0rM)repeat_interleavernr4rlrmrr}rr) rurvrwrxrjK_for_qk repeat_factor scale_factor sqrt_scaleQ_scaledK_scaleds r"r{r{sH222+/CC &&}!&<<$UAGAJ77L<((J:~H*$H <("4"4R"<"< = ==r$ Attention) is_causal kv_num_heads q_num_headsryrjsoftcapsoftmax_precisionV attn_maskpast_key past_valuerrrrrc  d\} }}t|j}|jd}t|jdkr`tj|dko|dkd|jd}t |||}t |||}t |||}tjt|jdko/t|jdkot|jdkd|j|}t | |} |tj||g| n|}|tj||g| n|}||}}|j| }|j| }|j|}|j|}| d ko"| dko| duo|dup|jtj k}t|||rqd}||jtj kr|n|}tj j ||||d || t||k }t||||| | }nY||kr3||z}||| }||| }tj|||j|j }|rutj|dud tjtj||tj |j }||t+d}|?|jtj kr%||t+d}n||z}t | |jd} t-j| }!||!z}"||!z}#tj|"|#dd}$|$}|$|z}%| dkr|%}| d kr| tj|%| z z}%| dkr|%}| x| t6vrX|%j}&|%t:j| }%tj|%d }'|'|&}'n-tj|%d }'ntj|%d }'| dkr|'}tj|'|}|dkr=|dd !||d}||||fS)zMAttention-23 https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23)r r1r?rr?cdS)Nz;q_num_heads and kv_num_heads must be provided for 3D inputsrrr$r"r5zattention_23..sQr$r rAcdS)Nz'Q, K, and V should be 4D tensors by nowrrr$r"r5zattention_23..!s9r$NrTr)r dropout_prrj enable_gqa)dtypedevicecdS)Nz'Cannot use both is_causal and attn_maskrrr$r"r5zattention_23..vs+Tr$z-infr0rMr1)"rVr4rrWrtrnr\clonerboolrnn functionalscaled_dot_product_attentionr~rzerosrtrilones masked_fillfloatrlrmr}rrtanh-_ATTENTION_23_ALLOWED_INTERMEDIATE_PRECISIONStorONNX_DTYPE_TO_TORCH_DTYPEsoftmaxrsrq)(rurvrrrrrrrryrjrr num_head_dim sequence_dimhead_diminput_shape_lenr8q_sequence_length q_head_size present_key present_valuerwrxkv_sequence_length can_use_sdpasdpa_attn_maskrh qk_outputr attn_bias causal_maskrrrrqk_matmul_output qk_with_biasoriginal_dtype qk_softmaxs( r" attention_23rs&,3(L,!'llOJ 17||q 1  2!2 Q Q   GAJ a[ 9 9 a\ : : a\ : : L AG Ec!'lla/ECLLA4E99 '(#K e[ 1 1E    8Q-\2222 WWYY  !  :q/|4444 WWYY  qA',/7<0 -. 3 A !Q & A  % A$  ?)/UZ"?  35IJJJr-  +4?ej+H+HiZZiN$AA $#';;B   1   !    "6 6 6/3GGM##M|#DDA##M|#DDAK 1    K LT!#T#T    * %&*8 K"--{lE&MMJJI  %*,,%119*eFmmLL & 1 ) ;; Y|,, z>z>!<(2D2DR2L2LMM% ()3 A % %$I S=="UZ w0F%G%GGL A % %$I  ( $QQQ!-!3+#=>OP   #]:: "]rsX ..........'''''' ******Yt__ WT]]ACUZ2H<=BBB09 11- !$ xB (2r6"223*  R  ,0 D/  !D/D/D/ |D/|D/|D/5<( D/  D/D/D/ \D/D/D/! D/NHXe_HHHHHH  L &) 69  \    F |F |FF F E? F  F \FFFF$47 > |> |>> > E? >  \ >>>>, +r )-'+)- F9!"!'+F9F9F9 |F9 |F9 |F9 % F9 u|$ F9 & F9F9F9F9F9 E?F9F9 }F9 5<u|U\ ABF9F9F9F9F9F9r$