%VjiX-ddlmZddlZgdZGddejjZGddejjZGdd ejjZGd d ejj Z Gd d ejj Z Gddejj Z Gddej jj jZGddejjZdS))warnN)ReLU6 HardswishELU LeakyReLUSigmoidSoftmaxMultiheadAttentionPReLUcHeZdZdZdfd ZdZdZeddZxZ S)raApplies the element-wise function: :math:`\text{ReLU6}(x) = \min(\max(x_0, x), q(6))`, where :math:`x_0` is the zero_point, and :math:`q(6)` is the quantized representation of number 6. Args: inplace: can optionally do the operation in-place. Default: ``False`` Shape: - Input: :math:`(N, *)` where `*` means, any number of additional dimensions - Output: :math:`(N, *)`, same shape as the input .. image:: ../scripts/activation_images/ReLU6.png Examples:: >>> m = nn.quantized.ReLU6() >>> input = torch.randn(2) >>> # xdoctest: +SKIP >>> input = torch.quantize_per_tensor(input, 1.0, 0, dtype=torch.qint32) >>> output = m(input) FcXt|||_dSN)super__init__inplace)selfr __class__s b/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/ao/nn/quantized/modules/activation.pyrzReLU6.__init__,s& !!! cVtjj||jSr)torchops quantizedrelu6rrinputs rforwardz ReLU6.forward0sy"(( ===rcdS)NQuantizedReLU6rs r _get_namezReLU6._get_name3rc*t|jSr)rr)moduse_precomputed_fake_quants r from_floatzReLU6.from_float6sS[!!!rF) __name__ __module__ __qualname____doc__rrr" staticmethodr' __classcell__rs@rrrs0>>>   """\"""""rrc^eZdZdZd fd ZdZdZed dZe dZ xZ S) rzThis is the quantized version of :class:`~torch.nn.Hardswish`. Args: scale: quantization scale of the output tensor zero_point: quantization zero point of the output tensor Nc||d}t|dtj|fi||dtj|fi|dSN)devicedtypescale zero_pointrrregister_bufferrtensor)rr5r6r3r4factory_kwargsrs rrzHardswish.__init__Cs|$*U;;  Wel5&K&KN&K&KLLL \5< +U+Un+U+UVVVVVrcbtjj||j|jSr)rrr hardswishr5r6rs rrzHardswish.forwardIs#y",,UDJPPPrcdS)NQuantizedHardswishr r!s rr"zHardswish._get_nameL##rFc|j\}}tt|t |Sr)activation_post_processcalculate_qparamsrfloatintr%r&r5r6s rr'zHardswish.from_floatOs77IIKKzus:777rcN|t|t|Sr)rCrDclsr%r5r6s rfrom_referencezHardswish.from_referenceTs s5<<Z111r)NNr( r)r*r+r,rrr"r-r' classmethodrIr.r/s@rrr;sWWWWWW QQQ$$$888\822[22222rrc^eZdZdZd fd ZdZdZed dZe dZ xZ S) rzThis is the quantized equivalent of :class:`~torch.nn.ELU`. Args: scale: quantization scale of the output tensor zero_point: quantization zero point of the output tensor alpha: the alpha constant ?cft|||_||_dSr)rrr5r6)rr5r6alphars rrz ELU.__init__bs-  $rctjjjj||j|j|jSr) raonnr functionalelur5r6rOrs rrz ELU.forwardgs3x{$/33 4:t    rcdS)N QuantizedELUr r!s rr"z ELU._get_namels~rFc|j\}}tt|t ||jSr)rArBrrCrDrOrEs rr'zELU.from_floatos;7IIKKz5<<Z#)<<(>???rr() r)r*r+r,rCrDrrrKr'r.r/s@rrrs3U3s333333    @@@[@@@@@rrc^eZdZdZd fd ZdZdZed d Ze d Z xZ S) r a,This is the quantized version of :class:`~torch.nn.Softmax`. Args: dim: A dimension along which Softmax will be computed (so every slice along dim will sum to 1). scale: quantization scale of the output tensor zero_point: quantization zero point of the output tensor NrMrcrt||_||_||_dSr)rrdimr5r6)rrnr5r6rs rrzSoftmax.__init__s2  $rc|j}|:d}tjjd||}tjj|||j|j S)Nsoftmax) rnrrRrS_get_softmax_dimrrrqr5r6)rrrn stacklevels rrzSoftmax.forwardsch ;J(%66599;; Cy"**5#tz4?SSSrcdS)NQuantizedSoftmaxr r!s rr"zSoftmax._get_names!!rFc|j\}}t|jt |t |Sr)rArBr rnrCrDrEs rr'zSoftmax.from_floats;7IIKKzswe c*oo>>>rcZ||jt|t|Sr)rnrCrDrGs rrIzSoftmax.from_references$s37E%LL#j//:::r)NrMrr(rJr/s@rr r s%%%%%% T T T"""???\?;;[;;;;;rr cleZdZejjjjZdZ e dZ e dZ dS)r cdS)NQuantizedMultiheadAttentionr r!s rr"zMultiheadAttention._get_names,,rc td)NzpIt looks like you are trying to convert a non-observed MHA module. Please, see the examples on quantizable MHAs.)NotImplementedError)rHothers rr'zMultiheadAttention.from_floats" 0   rctjj|dddd}||_|je|jd}tj|d\}}tj |||tj }t|d||j e|jd}tj|d\}}tj |||tj }t|d||` |`|S)NFT)mappingrremove_qconfigconvert_custom_config_dictbias_k) reduce_rangebias_v)rrQ quantizationconvertrr _parameterspop_choose_qparams_per_tensorquantize_per_tensorquint8setattrrin_proj_weight in_proj_bias)rHr} convertedrsczprs r from_observedz MultiheadAttention.from_observedsH)11 '+ 2  "    '*..x88F5f5QQQFB.vr2u|LLF Ix 0 0 0   '*..x88F5"FB.vr2u|LLF Ix 0 0 0  $  "rN) r)r*r+rrQrR quantizabler _FLOAT_MODULEr"rKr'rr rrr r shHK+>M---  [   [   rr c eZdZdZ ddedededdffd Zd ejddfd Z d ejdejfd Z d Z e ddZ e dZxZS)r a%This is the quantized equivalent of :class:`~torch.nn.PReLU`. Args: scale: quantization scale of the output tensor zero_point: quantization zero point of the output tensor num_parameters: number of parameters: 1, or the number of channels at input. Default: 1 rfrgnum_parametersr\Nc t||_||_||_t j|t j}t j|ddt j }| |dS)N)r4rMr)r5r6r4) rrrr5r6rrandnrCrr set_weight)rrfrgrwqwrs rrzPReLU.__init__!sz ,! + Kek : : :  &q V V V rrc||_dSr)weight)rrs rrzPReLU.set_weight,s  rrcntjj||j|j|jSr)rrrprelurr5r6rs rrz PReLU.forward/s.y"(( 4; DO   rcdS)NQuantizedPReLUr r!s rr"zPReLU._get_name4r#rFc6|j\}}|t|t||j}|j}|j}|||jtj krtd|j|\}} tj |t|t| tj } | | |SNz9PReLU's weight observer should have dtype quint8 but got ) rArBrCrDrrqconfigr4rrrrr) rHr%r&r5r6qprelufloat_wtobserverwt_scalewt_zpqweights rr'zPReLU.from_float7s7IIKKzU5\\3z??C4FGG:##%%;%%'' >U\ ) ) \HN\\   #4466%+ eHoos5zz5<   '""" rc|t|t||j}|j}|j}|||jt jkrtd|j| \}}t j |t|t|t j} | | |Sr) rCrDrrrr4rrrrBrr) rHr%r5r6rrrrrrs rrIzPReLU.from_referenceIsU5\\3z??C4FGG:##%%;%%'' >U\ ) ) \HN\\   #4466%+ eHoos5zz5<   '""" r)rr()r)r*r+r,rCrDrrTensorrrr"rKr'rIr.r/s@rr r sRS  ! 69 KN        ELT U\ el       ["[rr )warningsrr__all__rRReLUrrrrrr rQrr Moduler r rrrs    %"%"%"%"%"EHM%"%"%"P22222"222<=====%(,===@&S&S&S&S&S"&S&S&SR@@@@@eh@@@6$;$;$;$;$;eh$;$;$;N111110C111h@@@@@EHO@@@@@r