QVji0rddlZddlmZddlmZmZddlmZmZm Z m Z ddl Z ddl mZddlmZmZddlmZgdZe d d Zd efd ZedGddeeZedGddeZedGddeeZdS)N) defaultdict)IteratorSized)AnyCallableOptionalTypeVar)functional_datapipe) DataChunk IterDataPipe)_check_unpickable_fn)BatcherIterDataPipeGrouperIterDataPipeUnBatcherIterDataPipe_T_coT) covariantnamec|dvrQtjd|d|dtdttjjjjj |Stdtd|) N)SHARDING_PRIORITIESShardingFilterIterDataPipe`zc` from `torch.utils.data.datapipes.iter.grouping` is going to be removed in PyTorch 2.1Please use `z5` from the `torch.utils.data.datapipes.iter.sharding`)category stacklevelzmodule z has no attribute ) warningswarn FutureWarninggetattrtorchutilsdata datapipesitershardingAttributeError__name__)rs b/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/utils/data/datapipes/iter/grouping.py __getattr__r(s DDD  W W W W W W"     u{'16?FFF E8EEtEE F FFbatchc eZdZUdZeed<eed<eed<defdededede eddf fd Z de efd Z defd Z xZS) ra2 Creates mini-batches of data (functional name: ``batch``). An outer dimension will be added as ``batch_size`` if ``drop_last`` is set to ``True``, or ``length % batch_size`` for the last batch if ``drop_last`` is set to ``False``. Args: datapipe: Iterable DataPipe being batched batch_size: The size of each batch drop_last: Option to drop the last batch if it's not full wrapper_class: wrapper to apply onto each batch (type ``List``) before yielding, defaults to ``DataChunk`` Example: >>> # xdoctest: +SKIP >>> from torchdata.datapipes.iter import IterableWrapper >>> dp = IterableWrapper(range(10)) >>> dp = dp.batch(batch_size=3, drop_last=True) >>> list(dp) [[0, 1, 2], [3, 4, 5], [6, 7, 8]] datapipe batch_size drop_lastF wrapper_classreturnNc|dks Jdt||_||_||_||_dS)Nrz+Batch size is required to be larger than 0!)super__init__r,r-r.r/)selfr,r-r.r/ __class__s r'r3zBatcherIterDataPipe.__init__AsRA~~~L~~~   $"*r)c#Kg}|jD]H}||t||jkr||Vg}It|dkr |js||VdSdSdSNr)r,appendlenr-r/r.)r4r*xs r'__iter__zBatcherIterDataPipe.__iter__Os  A LLOOO5zzT_,,((///// u::>>> 0((/////// > 0 0r)ct|jtrJ|jrt |j|jzSt |j|jzdz |jzSt t|jd)Nz# instance doesn't have valid length) isinstancer,rr.r9r- TypeErrortyper&r4s r'__len__zBatcherIterDataPipe.__len__Zs| dmU + + Y~ U4=))T_<<DM**T_>> # xdoctest: +SKIP >>> from torchdata.datapipes.iter import IterableWrapper >>> source_dp = IterableWrapper([[[0, 1], [2]], [[3, 4], [5]], [[6]]]) >>> dp1 = source_dp.unbatch() >>> list(dp1) [[0, 1], [2], [3, 4], [5], [6]] >>> dp2 = source_dp.unbatch(unbatch_level=2) >>> list(dp2) [0, 1, 2, 3, 4, 5, 6] r=r, unbatch_levelc"||_||_dSN)r,rL)r4r,rLs r'r3zUnBatcherIterDataPipe.__init__|s  *r)c#bK|jD]$}|||jEd{V%dS)NrL)r,_diverL)r4elements r'r;zUnBatcherIterDataPipe.__iter__sS} M MGzz'9KzLL L L L L L L L L M Mr)c#K|dkrtd|dkrFt|ttfr$|D]}||dEd{V dS|VdS|dkr|VdSt|ttfr'|D]"}|||dz Ed{V#dSt d|jd)Nz unbatch_level must be -1 or >= 0rPrr=zunbatch_level z" exceeds the depth of the DataPipe) ValueErrorr>listr rQ IndexErrorrL)r4rRrLitems r'rQzUnBatcherIterDataPipe._dives: 2  ?@@ @ B  'D)#455 #BBD#zz$bzAAAAAAAAAABB a  MMMMM'D)#455 #QQD#zz$ma>OzPPPPPPPPPPQQ![T%7[[[r)N)r=) r&rCrDrEr rGr3r;rQr)r'rrdsf,+++c++++MMMr)rgroupbyceZdZdZdddddddeedeegefded e d e e d e e d efd Z dZ dZ ddZdZdZdZdS)ra Groups data from IterDataPipe by keys from ``group_key_fn``, yielding a ``DataChunk`` with batch size up to ``group_size``. (functional name: ``groupby``). The samples are read sequentially from the source ``datapipe``, and a batch of samples belonging to the same group will be yielded as soon as the size of the batch reaches ``group_size``. When the buffer is full, the DataPipe will yield the largest batch with the same key, provided that its size is larger than ``guaranteed_group_size``. If its size is smaller, it will be dropped if ``drop_remaining=True``. After iterating through the entirety of source ``datapipe``, everything not dropped due to the buffer capacity will be yielded from the buffer, even if the group sizes are smaller than ``guaranteed_group_size``. Args: datapipe: Iterable datapipe to be grouped group_key_fn: Function used to generate group key from the data of the source datapipe keep_key: Option to yield the matching key along with the items in a tuple, resulting in `(key, [items])` otherwise returning [items] buffer_size: The size of buffer for ungrouped data group_size: The max size of each group, a batch is yielded as soon as it reaches this size guaranteed_group_size: The guaranteed minimum group size to be yielded in case the buffer is full drop_remaining: Specifies if the group smaller than ``guaranteed_group_size`` will be dropped from buffer when the buffer is full Example: >>> import os >>> # xdoctest: +SKIP >>> from torchdata.datapipes.iter import IterableWrapper >>> def group_fn(file): ... return os.path.basename(file).split(".")[0] >>> source_dp = IterableWrapper( ... ["a.png", "b.png", "a.json", "b.json", "a.jpg", "c.json"] ... ) >>> dp0 = source_dp.groupby(group_key_fn=group_fn) >>> list(dp0) [['a.png', 'a.json', 'a.jpg'], ['b.png', 'b.json'], ['c.json']] >>> # A group is yielded as soon as its size equals to `group_size` >>> dp1 = source_dp.groupby(group_key_fn=group_fn, group_size=2) >>> list(dp1) [['a.png', 'a.json'], ['b.png', 'b.json'], ['a.jpg'], ['c.json']] >>> # Scenario where `buffer` is full, and group 'a' needs to be yielded since its size > `guaranteed_group_size` >>> dp2 = source_dp.groupby( ... group_key_fn=group_fn, ... buffer_size=3, ... group_size=3, ... guaranteed_group_size=2, ... ) >>> list(dp2) [['a.png', 'a.json'], ['b.png', 'b.json'], ['a.jpg'], ['c.json']] Fi'N)keep_key buffer_size group_sizeguaranteed_group_sizedrop_remainingr, group_key_fnr\r]r^r_r`cNt|||_||_||_||_t t |_d|_||_ d|_ ||d|cxkr|ksnJ||_ ||d|cxkr|ksnJ||_ ||_ t|_ dSr7)r r,rar\max_buffer_sizerrVbuffer_elementscurr_buffer_sizer^r_r`r r/)r4r,rar\r]r^r_r`s r'r3zGrouperIterDataPipe.__init__s \***  (  *7B47H7H !$%)"  !k&=z0000[000000)3D & ,)a2G.U.U.U.U:.U.U.U.U.UU)>D &,&r)cd}d}d}|jD]<}t|j||krt|j|}|}=|j:||jkr/|js(t dt |j||j ||jkr |j|}|xj|zc_|j|=|S)NrzFailed to group items)rdkeysr9r_r` RuntimeErrorstrre)r4 biggest_key biggest_sizeresult_to_yieldfindkeys r'_remove_biggest_keyz'GrouperIterDataPipe._remove_biggest_keys  +0022 & &G4'011L@@"4#7#@AA %   & 2t999':'T-A+-N)O)O   & .t999"2;?O -   -r)c#VK|jD]}||}|j|||xjdz c_|j|jt |j|kr_||j|}|jr||fn|V|xjt |j|zc_|j|=|j|j kr:| }|$||}|jr||fn|Vt|j D][}||j |}|xjt |zc_|jr||fn|V\dS)Nr=)r,rardr8rer^r9r/r\rcrntuplergpop)r4r:keyresultrls r'r;zGrouperIterDataPipe.__iter__ s E EA##A&&C   % , ,Q / / /  ! !Q & ! !*t#$S)CC00*.););Drs######++++++++333333333333////EEEEEEGGGGGGGGHHHHHH    4((( Gc G G G GW;Y;Y;Y;Y;Y,y1;Y;Y;Y|Y11111L11 1hYo%o%o%o%o%,y1o%o% o%o%o%r)