&Vji}?ddlZddlZddlmZddlZddlmZddlm Z m Z ddl m Z mZgdZejddd gZd Zd Zejd i aet*_t*jr et*_dZdZe je jfdZdZdZe jddfdZ dZ!dZ" ddZ#dZ$dZ%dZ&dZ'dZ(ede e(dS)N)cast)api constants)_group_membership_management_update_group_membership)backend_registeredregister_backendconstruct_rpc_backend_options init_backend BackendValue BackendTyper %construct_rpc_backend_options_handlerinit_backend_handlercd|jzS)Nz BackendType.)name)selfs `/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/distributed/rpc/backend_registry.py_backend_type_reprrs DI %%z An enum class of available backends. PyTorch ships with a builtin ``BackendType.TENSORPIPE`` backend. Additional ones can be registered using the :func:`~torch.distributed.rpc.backend_registry.register_backend` function. rvaluenamescB|tjvS)z Checks if backend_name is registered as an RPC backend. Args: backend_name (str): string to identify the RPC backend. Returns: True if the backend has been registered with ``register_backend``, else False. )r __members__keys) backend_names rr r 3s ;27799 99rcFt|rtd|ddtD}t|t ||ifi|}t jd|att_tj rtt_ t|S)aRegisters a new RPC backend. Args: backend_name (str): backend string to identify the handler. construct_rpc_backend_options_handler (function): Handler that is invoked when rpc_backend.construct_rpc_backend_options(**dict) is called. init_backend_handler (function): Handler that is invoked when the `_init_rpc_backend()` function is called with a backend. This returns the agent. z RPC backend z: already registeredc(i|]}|j|jS)rr).0members r z$register_backend..RsNNN&+v|NNNr)rrrr) r RuntimeErrorrdictr enumEnumr__repr____doc___backend_type_doc)rrrexisting_enum_dictextended_enum_dicts rr r @s,''NL,LLLMMMNN+NNN ,6[%9  )-7IJJJK-K0/  | $$rc *|jj||fi|SN)rr)backend rpc_timeout init_methodkwargss rr r es/ ?7= >[  $*  rc&|jj|i|Sr.)rr)r/argsr2s rr r ps -7= -t >v > >>rcxtj}tj||||}| Jd|dkr?||kr't d|d||dkr?||kr't d|d||S)Nz*Failed to initialize default ProcessGroup.zrank argument z doesn't match pg rank zworld_size argument z doesn't match pg size ) rpc_constantsDEFAULT_PROCESS_GROUP_TIMEOUTdistProcessGroupGloorankr$size)storer; world_sizeprocess_group_timeoutgroups r_init_process_grouprAts)G  !%z;P Q QE   J     --WDWWWWXXXbzUZZ\\99 T: T Tejjll T T    Lrc .ddlm}||||||S)Nr)TensorPipeRpcBackendOptions)r0r1num_worker_threads _transports _channels)rC)r0r1rDrErFr2rCs r1_tensorpipe_construct_rpc_backend_options_handlerrHs?.----- & &-    rc:tfd|DS)Nc3nK|]/}|jdkp|jdkod|jcxkokncV0dS)cpucudarN)typeindex)r!d device_counts r z/_tensorpipe_validate_devices..sl  %MAFf,Lag1L1L1L1L 1L1L1L1Lr)all)devicesrPs `r_tensorpipe_validate_devicesrTs;   rc\dt|D}tj|||||f|d|D}d|D}d|D}d|D} t |||| t |||} t ||| }| |fS)Ncg|]}ddigf S)rGrr )r!_s r zB_tensorpipe_exchange_and_check_all_device_maps..s 666Q"aR666rcg|] \}}}}| Sr r )r!rrWs rrXzB_tensorpipe_exchange_and_check_all_device_maps..s 444-$1a444rci|] \}}}}|| Sr r )r!rcountrWs rr#zB_tensorpipe_exchange_and_check_all_device_maps..s#GGG):uauGGGrci|] \}}}}|| Sr r )r!rrWmap_s rr#zB_tensorpipe_exchange_and_check_all_device_maps..s#CCC&6dAtQtTCCCrci|] \}}}}|| Sr r )r!rrWrSs rr#zB_tensorpipe_exchange_and_check_all_device_maps..s#EEE%8T1a4EEEr)ranger<r9all_gather_object_validate_device_maps_create_reverse_mapping_create_device_list) my_namemy_device_countmy_device_maps my_devicesr@gathered all_namesall_device_countsall_device_maps all_devicesreverse_device_mapss r._tensorpipe_exchange_and_check_all_device_mapsrns 76% "5"5666  7O^ZH%548444IGGhGGGCC(CCCOEEHEEEK)%6UUU2'9oVV$ZATUUJ  **rTc |D]}||}tt|t|krtd|d|t|||std|d|d|||D]U}|rjt|||s0td|d||d|||D]\}} tt| t| krtd|d|d| ||r\t| ||s!td|d |d| d ||nIt| ||s!td|d |d| d||||gr]t| ||s!td|d |d| d ||}||vrIt| ||s!td|d |d| d||̐WdS)NzNode z" has duplicated devices devices = z, has devices with invalid indices devices = z device count = z@ has invalid target node names in its device maps device maps = z node names = z5 has duplicated target devices in its device map for z device map = z5 has unexpected source devices in its device map for z devices = z? has source devices with invalid indices in its device map for z5 has unexpected target devices in its device map for z? has target devices with invalid indices in its device map for ) lenset ValueErrorrTrissubsetitemsvaluesget) rirjrkrlis_static_groupnoderS source_node target_noder]s rraras?  d# s7||  G , ,WTWWgWWXX X+G5Ft5LMM <<<$<<"3D"9<<  !22  3{'C'H'H'J'J#K#K#T#T $ $  , ,,!0!=!B!B!D!D,, ),,  "1!=!C!C!E!E( (  K3t{{}}%%&&#d))33 +K++-8++$(++ ;' 499;;''00JJ$@ @@1<@@(,@@&1%=@@2 .{; !GKGG-8GG$(GG'8 &DGG {B// 4;;==))22;{3KLL$@ @@1<@@(,@@&1%=@@ 111:V 0=;;1!GKGG-8GG$(GG'8 &DGGG( 22rc|st}|D])}||*|D])}||*|t jdt|}t|d}|S)NrKc|jSr.)rN)rOs rz%_create_device_list..s!'r)key) rqruupdaterdiscardtorchdevicelistsorted)rgrfrm devices_setr]s rrcrcs '), "))++ , ,D   tyy{{ + + + +'..00 , ,D   tyy{{ + + + +EL//000+&&  (9(9:::J rci}|D]9}|||vr-d|||D||<:|S)Nci|]\}}|| Sr r )r!kvs rr#z+_create_reverse_mapping.. s+)))A1)))r)rt)rdrirkrmrxs rrbrb sjGI od+ + +))!0!6w!?!E!E!G!G)))  % rcXddlm}t|tj}|}t j}t j r%|j rt j ||j |j fS)NrTensorPipeAgent) rGrrr_get_current_rpc_agent_get_backend_optionsrrLrP is_availablerSinit device_maps)ragentoptsrPs r_get_device_infosrs!!!!!! #"<">"> ? ?E  % % ' 'D:**,,L z  T\  )4< 77rc ddlm}t||}|}|j}|}iiigf\}}}}|D]} | j} | |krt j| t\} } } n@| }tj |j |j} } } | || <| || <| || <|| t!||||dt#|||}|D]H} t%|| || ||| <t j| t&||| |dfIdS)NrrF)rwT)r4)rGrrget_worker_inforget_worker_infosrrpc_syncrrrrLrPrrSappendrarbrcr)rrmy_worker_infordall_worker_infosrjrkrlri worker_info worker_namerP device_maprSrrms r#_set_devices_and_reverse_device_maprs!!!!!! % ( (E**,,N!G--//ACRR> Y'&& !& ' ! !03 .11 -L*gg--//D ''))  '.*L *6+&'1 $#* K %%%% 2'9oVV!   #6  $ok&BDW$ $ K   $ +k":rpc_backend_optionsrrCrPrwr@rmrSrs r _tensorpipe_init_backend_handlerrRs?>>>>>>> eTZ ( (CA%AABBB )+F G G  cNa c c   :**,,L(3ddeOD$E4<<'U    +  '  ( ( $W : " " $ $   JOO              U### &9&EFFFF   *%t < <  $O#E   ' ' ' 4E::::     +                  s*%&G  FG F;;G  GG TENSORPIPE)T)) collectionsr&typingrrtorch.distributed distributedr9rGrrr7_utilsrr__all__ namedtupler rr*r'rr(r)r r DEFAULT_RPC_TIMEOUT_SECDEFAULT_INIT_METHODr r rADEFAULT_NUM_WORKER_THREADSrHrTrnrarcrbrrrr rrrs/  --------JJJJJJJJ   &{%<>TU &&&dim2666 ) ,+K : : :"%"%"%N51???,%? &+++.QU@@@@F   8881 1 1 hUUUp5$r