/;jiIddlmZmZmZddlZddlmcmZddlmZddl m Z m Z ddl m Z e rddlmZmZe jeZdZd Zdd efd ZGd dejjZGddejZGddejZGddejZGddejZdS))OptionalTupleUnionN)nn) deprecatelogging)is_transformers_available) CLIPTextModelCLIPTextModelWithProjectionc g}t|ttfrIt|jjjD])\}}d|d}|j}|||f*ntd|j j |S)Ntext_model.encoder.layers.z .self_attnz.do not know how to get attention modules for: ) isinstancer r enumerate text_modelencoderlayers self_attnappend ValueError __class____name__) text_encoder attn_modulesilayernamemods O/root/voice-cloning/.venv/lib/python3.11/site-packages/diffusers/models/lora.pytext_encoder_attn_modulesr )sL,0K LMMm!,"9"A"HII - -HAu====D/C   s , , , , - k,J`Jikklll c g}t|ttfrIt|jjjD])\}}|j}d|d}|||f*ntd|j j |S)Nrz.mlpz(do not know how to get mlp modules for: ) rr r rrrrmlprrrr)r mlp_modulesrrmlp_modrs rtext_encoder_mlp_modulesr&7sK,0K LMMg!,"9"A"HII 0 0HAuiG7777D   g / / / / 0 eLDZDceefff r!? lora_scalecRt|D]O\}}t|jtr0||j_||j_||j_||j_Pt|D]7\}}t|j tr||j _||j _8dSN) r rq_projPatchedLoraProjectionr(k_projv_projout_projr&fc1fc2)rr(_ attn_module mlp_modules radjust_lora_scale_text_encoderr5Es3LAA99; k(*? @ @ 9,6K  ),6K  ),6K  ).8K +1,??33 : jn&; < < 3(2JN %(2JN %33r!cHeZdZd fd Zddddfd Zdd Zd Zd ZxZS)r,Nc2d}tdd|tddlm}||_|jjj}||jjj}||jj |jj |||||_ ||_ dS)NzsUse of `PatchedLoraProjection` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r,1.0.0r)LoRALinearLayer) network_alphadevicedtyperank) rsuper__init__ models.lorar;regular_linear_layerweightr=r> in_features out_featureslora_linear_layerr() selfrCr(r<r?r>deprecation_messager;r=rs rrAzPatchedLoraProjection.__init__TsT)74GHHH 111111$8!*18 =-4:E!0  % 1  % 2' " " " %r!F destinationprefix keep_varscp|j|jj||||dStj||||dS)NrK)rGrC state_dictr@)rHrLrMrNargsrs rrPz PatchedLoraProjection.state_dictosW  ! )74,7;v "uww!4[[deeeer!r'c v|jdS|jjjj|jjjj}}|jjj}|jjjj}|jjjj}|jj ||jj z|jj z }||tj |dddf|dddfdzz}|rKtj |rt!d|d||||jj_d|_||_||_||_dSNraThis LoRA weight seems to be broken. Encountered NaN values when trying to fuse LoRA weights for .LoRA weights will not be fused.r=r>)rGrCrDdatar>r=floatupdownr<r?torchbmmisnananyitemrtocpuw_upw_downr( rHr( safe_fusingr>r=w_origrbrc fused_weights r _fuse_loraz PatchedLoraProjection._fuse_loraws  ! ) F18=CTE^EeEjEqv*16<<>>%(/4::<<',38>>@@  ! / ;$0>>AWA\\DeiT111W vdTUTUTUg.W.WXY.Z!Z[  5;|4488::??AA 2OS222  1=vUZ0[0[!(-"&HHJJ jjll $r!c >t|ddt|dddS|jjj}|j|j}}|j|}|j |}||j tj |dddf|dddfdzz }||||jj_d|_d|_ dSNrbrcr=rrV) getattrrCrDrWr>r=rbr`rXrcr(r[r\rHrgr>r=rbrcunfused_weights r _unfuse_loraz"PatchedLoraProjection._unfuse_lorasfd++7GD(TX0A0AW\0A0]0]!(-  r!c|jd|_|j||S|||j||zzS)Nr')r(rGrC)rHinputs rforwardzPatchedLoraProjection.forwards] ? "!DO  ! ),,U33 3((//4?TE[E[\aEbEb3bccr!)r7Nr8Nr'F) r __module__ __qualname__rArPrhrorr __classcell__rs@rr,r,Ss%%%%%%6-1ufffffff%%%%@ dddddddr!r,ceZdZdZ ddedededeedeeej e fd eej f fd Z d ej d ej fd ZxZS)r;a A linear layer that is used with LoRA. Parameters: in_features (`int`): Number of input features. out_features (`int`): Number of output features. rank (`int`, `optional`, defaults to 4): The rank of the LoRA layer. network_alpha (`float`, `optional`, defaults to `None`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning device (`torch.device`, `optional`, defaults to `None`): The device to use for the layer's weights. dtype (`torch.dtype`, `optional`, defaults to `None`): The dtype to use for the layer's weights. r8NrErFr?r<r=r>ctd}tdd|tj||d|||_tj||d|||_||_||_||_ ||_ tj |jj d|z tj |jj dS)NzmUse of `LoRALinearLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r;r:F)biasr=r>r7std)r@rArrLinearrZrYr<r?rFrEinitnormal_rDzeros_) rHrErFr?r<r=r>rIrs rrAzLoRALinearLayer.__init__s N#W.ABBBIk4eFRWXXX )D,U6QVWWW+ (&  (a$h777 tw~&&&&&r! hidden_statesreturnc|j}|jjj}|||}||}|j||j|jz z}||Sr*r>rZrDr`rYr<r?rHr orig_dtyper>down_hidden_statesup_hidden_statess rrrzLoRALinearLayer.forward|"(   &!YY}'7'7'>'>??77#566   )  2TY > > "":...r!)r8NNN)rrtru__doc__intrrXrr[r=strr>rATensorrrrvrws@rr;r;s0)-59'+'''' '   ' u|S012 ' $''''''4 /U\ /el / / / / / / / /r!r;ceZdZdZ ddededed eeeeeffd eeeeeffd eeeeefefd ee ffd Z de j de j fdZ xZS)LoRAConv2dLayera" A convolutional layer that is used with LoRA. Parameters: in_features (`int`): Number of input features. out_features (`int`): Number of output features. rank (`int`, `optional`, defaults to 4): The rank of the LoRA layer. kernel_size (`int` or `tuple` of two `int`, `optional`, defaults to 1): The kernel size of the convolution. stride (`int` or `tuple` of two `int`, `optional`, defaults to 1): The stride of the convolution. padding (`int` or `tuple` of two `int` or `str`, `optional`, defaults to 0): The padding of the convolution. network_alpha (`float`, `optional`, defaults to `None`): The value of the network alpha used for stable learning and preventing underflow. This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning r8r7r7rNrErFr? kernel_sizestridepaddingr<ctd}tdd|tj|||||d|_tj||ddd|_||_||_tj |jj d|z tj |jj dS) NzmUse of `LoRAConv2dLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rr:F)rrrrzr)rrrzr7r{) r@rArrConv2drZrYr<r?r~rrDr) rHrErFr?rrrr<rIrs rrAzLoRAConv2dLayer.__init__s N#W.ABBBIk4[QWahotuuu )D,F6X]^^^+   (a$h777 tw~&&&&&r!rrc|j}|jjj}|||}||}|j||j|jz z}||Sr*rrs rrrzLoRAConv2dLayer.forwardrr!)r8rrrN)rrtrurrrrrrrXrAr[rrrrvrws@rrrs439.445)-'''' ' 3c3h/0 ' c5c?*+ 'sE#s(OS01' ''''''8 /U\ /el / / / / / / / /r!rceZdZdZdddeeffdZdeefdZdd ed e fd Z d Z dd e j dede j fdZxZS)LoRACompatibleConvz; A convolutional layer that can be used with LoRA. N lora_layerrcnd}tdd|tj|i|||_dS)NzpUse of `LoRACompatibleConv` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rr:rr@rArrHrrQkwargsrIrs rrAzLoRACompatibleConv.__init__0sGQ&1DEEE$)&)))$r!c:d}tdd|||_dSNznUse of `set_lora_layer()` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.set_lora_layerr:rrrHrrIs rrz!LoRACompatibleConv.set_lora_layer7s*O"G-@AAA$r!r'Fr(rec|jdS|jjj|jjj}}|jj}|jjjj}|jjjj}|jj||jjz|jj z }tj | d| d}| |j}|||zz} |rKtj| rt%d|d| |||j_d|_||_||_||_dS)Nr7 start_dimrTrUrV)rrDrWr>r=rXrYrZr<r?r[mmflattenreshapeshaper]r^r_rr`rarbrc _lora_scale) rHr(rer>r=rfrbrcfusionrgs rrhzLoRACompatibleConv._fuse_lora=s ? " F (. 0@0Gv!''))!(-3355%,17799 ? ( 4$/77$/:NND$,,,33V^^a^5P5PQQ//f!45  5;|4488::??AA 2OS222  (??&?FF HHJJ jjll %r!ct|ddt|dddS|jj}|jj|jj}}|j||_|j||_tj |j d|j d}| |j }||j|zz }||||j_d|_d|_dS)Nrbrcrkr7rrV)rlrDrWr>r=rbr`rXrcr[rrrrr)rHrgr>r=rrns rrozLoRACompatibleConv._unfuse_lora_s'fd++7GD(TXr=rXrYrZr<r?r[r\r]r^r_rr`rarbrcrrds rrhzLoRACompatibleLinear._fuse_loras ? " F (. 0@0Gv!''))!(-3355%,17799 ? ( 4$/77$/:NNDeiT111W vdTUTUTUg.W.WXY.Z!Z[  5;|4488::??AA 2OS222  (??&?FF HHJJ jjll %r!c *t|ddt|dddS|jj}|j|j}}|j|}|j|}||j tj |dddf|dddfdzz }||||j_d|_d|_dSrj) rlrDrWr>r=rbr`rXrcrr[r\rms rroz!LoRACompatibleLinear._unfuse_lorasfd++7GD(TXrs0*))))))))) &&&&&&&&::::::HGGGGGGGG  H % %       3 3U 3 3 3 3YdYdYdYdYdEHOYdYdYdx9/9/9/9/9/bi9/9/9/x=/=/=/=/=/bi=/=/=/@TOTOTOTOTOTOTOTOnGGGGG29GGGGGr!