&Vjib ddlZddlZddlZddlZddlmZmZmZddlm Z m Z m Z m Z m Z mZddlZddlmZddlmcmcmZddlmcmcmZddlmcmcmZddlmZddlm Z ddl!m"Z"m#Z#ddl$m%Z%ddl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-ddl.m/Z/m0Z0m1Z1m2Z2dd l3m4Z4dd l5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;mm?Z?dd l@mAZAdd lBmCZCddlDmEZEe rddlFmGZGdZH ddlImJZJmKZKn #eL$rdZHYnwxYweMdZNdZOePejQejQfZRe eejQeRfZSe;jTe2jTe;jUe2jUe;jVe2jVe;jWe2jWe;jXe2jXiZYe;jWe;jXgZZe;jVe;jXfZ[e d_de(deSde;de e?de e#de(f dZ\e de(deSde#de(fdZ]e de de^fdZ_e de#de^fdZ`e deMdejQfd Zae d!ejQdeMdejQfd"Zbd!ejQdeMdePejQejQffd#Zce d_de(d$ejdd%e eejjdd&ee eejjee eejjdfde(f d'Zfd&ege d(e^ddfd)Zhe de(d$ejdd*eiejed+e eeMejjfde(f d,Zke de(d$ejdde(fd-Zle de(de e;d.e e:d/e e7d0e^d1e^d2eMd3eMde(fd4Zme de(de(fd5Zne de(d6e6d7e^de(fd8Zoe d_de(de#de(fd9Zpe de(de(fd:Zqd$ejdd;egejeddfd<Zre de(d=ejdd+e eeMejjfd>e e ejdgdfd?e^de(f d@Zse de(d;egejed=ejdfdAZtdBejddCe eejjddeiejdfdDZu d_dBejjdd%eiejjddEe eejjedeiejjefdFZvdBejjdd%eiejjddeiewfdGZxdBejddeiewfdHZyd$ejdd*eiejed+e eeMejjfddfdIZzd+e eeMejjfdJeMdKe'de ejjfdLZ{d$ejdd*eiejed%eiejddePe^e^ffdMZ|dBejdd>e ejdgdfd%eiejdddfdNZ}dBejddOe ejjd%eiejddKe'fdPZ~dBejdd%eiejddegejdfdQZd$ejdd*eiejedReiejdOe ejjddf dSZd;egejedTegejdOe ejjddfdUZdVZd$ejdd*eiejedOe ejjdJeMdKe'dejjf dWZd$ejdd;egejedejQddfdXZdYegejddfdZZd$ejdd*eiejedeejefd[Zd*eiejeddfd\Zde;fd]ZdejQde jfd^ZdS)`N) GeneratorIterableIterator)AnyCallable no_type_checkOptional TYPE_CHECKINGUnion) default_hooks)_mesh_resources DeviceMesh)_get_default_group)_FSDPDeviceHandle _FSDPState_get_module_fsdp_state_is_fsdp_flattened!_named_parameters_with_duplicatesclean_tensor_name TrainingState)_FSDP_USE_FULL_PREC_IN_EVAL FlatParameterFlatParamHandleHandleShardingStrategy)_FreeEventQueue)BackwardPrefetch CPUOffloadFullOptimStateDictConfigFullStateDictConfigMixedPrecisionShardingStrategyStateDictConfig StateDictType)_Policy)DTensorExtensions)_sync_params_and_buffers)is_traceable_wrapper_subclass)RemovableHandleT) deferred_initfakeFi _fsdp_syncedstate process_groupsharding_strategypolicy device_meshreturnc8||td|tv}|r+|||td|dt|||}n<|r#||_|d|_n||n t |_|j|_|j|_ |j }|r||j z}tj ||_||jz |_|S)NzcCannot pass both process_group and device_mesh at the same time. Please just pass only one of them.zManual wrapping with zA requires explicit specification of process group or device_mesh.rmesh_dim) ValueErrorHYBRID_SHARDING_STRATEGIES*_init_process_group_state_for_hybrid_shard _device_mesh get_groupr-rranksize world_size_inter_node_pgr DefaultState_get_gradient_predivide_factor_gradient_predivide_factor_gradient_postdivide_factor)r,r-r.r/r0is_hybrid_strategydata_parallel_world_sizes \/root/voice-cloning/.venv/lib/python3.11/site-packages/torch/distributed/fsdp/_init_utils.py_init_process_group_staterEYsw [%< <   +.HH  V^ 8KS(9SSS  ?}kEE  !,E "-"7"7"7"C"CE  "/!: @R@T@T  $))++EJ*//11E$/@ E$8$=$=$?$?? "AA $   $ !5#CC % Lc|rdt|r>||_|d|_|d|_nt d|j|Gt}t||j \}}||_||_n>t|r|\|_|_nt dt|t|j|_|S)Nrr3z,Expected device_mesh to have ndim=2 but got zmExpected process_group to be passed in as either None or Tuple[dist.ProcessGroup, dist.ProcessGroup] but got r-)"_is_valid_hybrid_shard_device_meshr8r9r=r-r5ndimr!_init_intra_and_inter_node_groups_device_handle device_count_is_valid_hybrid_shard_pg_typetype_get_default_comm_hook_state_inter_node_state)r,r-r0 default_groupintra_node_groupinter_node_groups rDr7r7sJ  -k : : !,E $/#8#8!#8#D#DE "-"7"7"7"C"CE  Q{?OQQ   *,, -N 5/<<>>. . **// *- 8 8 9F 5E !5!5]GKMGZGZ]]  ;*E LrFct|to+t|dkotd|DS)Nc3JK|]}t|tjVdSN) isinstancedist ProcessGroup).0pgs rD z1_is_valid_hybrid_shard_pg_type..s/JJb 2t011JJJJJJrF)rZtuplelenallrIs rDrOrOsL =%(( K   ! # K JJMJJJ J JrFcBt|to |jdkS)NrW)rZrrK)r0s rDrJrJs k: . . H;3Cq3HHrFnum_devices_per_nodec4tj|\}}|S)aU Return a process group across the current node. For example, given each row is a distinct node: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 This API would return an intra-node subgroup across [0, 1, ..., 7] or [8, 9, ..., 15] depending on the process's rank. For example, rank 3 would get [0, 1, ..., 7]. )r[ new_subgroups)rdintra_node_subgroup_s rD_init_intra_node_process_groupris!"/0DEE rFglobal_process_groupcL d}tj|}tj|}|z}tj|z}t D]< fdt |D}tj||} |kr|}=| J|d|S)a Return an inter-node process group where each contained rank has the same local rank. For example, given each row is a distinct node: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 This API would return inter-node process group [0, 8], [1, 9], [2, 10], and so forth depending on the process's rank. For example, rank 1 would get [1, 9], rank 5 would get [5, 13]. Nc g|] }|zz Srm)r]i local_rankrds rD z2_init_inter_node_process_group..s1! ! ! 89J!22 3! ! ! rF)ranksbackendz. expected to assign inter-node pg, but did not)r[ get_backendget_world_sizeget_rankrange new_group) rjrd inter_node_pgsharding_backendr< num_nodes my_local_rankranks_for_inter_groupgrpros ` @rD_init_inter_node_process_groupr~s M'(<==$%9::J22IM"677:NNM011  ! ! ! ! ! =B9=M=M! ! ! n#8BRSSS  & &M  $ $ HHH % $ $ rFc@t|t||fS)a Initialize intra and inter-node process groups and return the ones corresponding to this process's rank. This function can be used to initialize process groups for ``HYBRID_SHARD`` or ``_HYBRID_SHARD_ZERO2`` in FSDP. This function assumes each node has an equal number of CUDA-enabled devices. Returns: Tuple[dist.ProcessGroup, dist.ProcessGroup]: Intra and inter-node process group. )rir~)rjrds rDrLrLs) '';<<&';=QRR rFmoduleignored_modulesignored_statesc||tdd}|du}|r t|}t|dn#g}t|t|ngdt|dkr%t |dt jr|}n|}t|||_t||j||_ t||j|_ |S)NzfCannot pass both ignored_modules and ignored_states at the same time. Please just pass ignored_states.TFr) r5list_check_ignored_statesrarZnn Parameter_get_ignored_modules_ignored_modules_get_ignored_params_ignored_params_get_ignored_buffer_names_ignored_buffer_names)r,rrrignored_parameterspassed_as_ignored_statesignored_states_lists rD_init_ignored_module_statesrs$"~'A :   -T9 ">22148888 %4%@D ! ! !b%    !## )!,bl ; ; 2!4  1O1&/JJE/ E #< ##E LrFrct|dkrdS|rltd|D}td|D}|s4|s4td|Dt}t d|dSdStd|Ds2td |Dt}t d |dS) z Check that the ignored states are uniformly parameters or uniformly modules. We may remove this check in the future if we permit mixing. rNc3JK|]}t|tjVdSrY)rZrrr]r,s rDr_z(_check_ignored_states..Is.UUUE2<88UUUUUUrFc3JK|]}t|tjVdSrYrZrModulers rDr_z(_check_ignored_states..Js.SS5*UBI66SSSSSSrFc,h|]}t|SrmrPrs rD z(_check_ignored_states..M"K"K"K54;;"K"K"KrF)keyzUignored_states expects all nn.Parameter or all nn.Module list elements but got types c3JK|]}t|tjVdSrYrrs rDr_z(_check_ignored_states..Ss.LLE:eRY//LLLLLLrFc,h|]}t|Srmrrs rDrz(_check_ignored_states..TrrFz>ignored_modules expects nn.Module list elements but got types )rarbsortedreprr5)rr all_params all_modules sorted_typess rDrr>s6 >aUUnUUUUU SSNSSSSS  + !"K"KN"K"K"KQUVVVL9*699     LL^LLLLL !"K"KN"K"K"KQUVVVL(%((   rFignored_params device_idcd}|0t|tjr|ntj|}|t||D]T}|jjdvr||j}|jj|jkr$t d|jd|jjU|ptj}|jdkrt dtj ||_ |S)a= Determine device handle used for initializing FSDP. If a device is specified by ``device_id``, then returns device handle corresponds to that device type. Otherwise, If the module is already on a non-CPU device, then the device type is that non-CPU device type. If the module is on CPU or meta, then the device type is the current accelerator device. See the :ref:`Accelerators` for details. This method will be called once ignored parameters was determined, as the device handle maybe needed for other initialization. N>cpumetazLFSDP does not support modules with different device types but got params on z and rzOFSDP needs a non-CPU accelerator device, but no accelerator device is detected.) rZtorchdevice_get_orig_paramsrP RuntimeError_C_get_acceleratorr from_devicerM)r,rrrdetermined_deviceparams rD_init_device_handler[s1()U\22 )IIi((   %fn==  E| O33 ($)L!!<$(9(>>>&^->-C^^JO,J[^^? .L1J1J1L1L  !U * *a -89JKKE LrFct||_i}|D]\}}t|}|j||<||_|SrY)_get_buffer_names _buffer_names named_buffersrdtype_buffer_name_to_orig_dtype)r,rr buffer_namebuffers rD_init_buffer_statersf ,F33E :<%3355?? V' 44 28,";//'AE$ LrFmixed_precision cpu_offloadlimit_all_gathersuse_orig_paramsbackward_prefetch_limitforward_prefetch_limitc^|jdkrA|tjkr$tjd|p tjdtj}n,|tjkrtjdt d|p tj|_|p t|_ |4tj dt|j tjt"dd k|_|p t'|_||_||_t.j|_d|_t7|_t;j|_tAj!|j|||_"d|_#i}||_$d} | |_%g} | |_&|S) NrHz/FSDP is switching to use `NO_SHARD` instead of z since the world size is 1.zoThe `NO_SHARD` sharding strategy is deprecated. If having issues, please use `DistributedDataParallel` instead.) stacklevelz'torch.distributed.fsdp.mixed_precision.1)'r<r!NO_SHARDwarningswarn FULL_SHARD FutureWarningr.r rrr_log_api_usage_oncestrosenvirongetr_use_full_prec_in_evalrrr_use_orig_paramsrIDLEtraining_state_is_rootr_free_event_queuer[get_debug_level _debug_levelexec_order_utils_ExecOrderData_exec_order_data_unshard_event_fully_sharded_module_to_handle_handleparams) r,r.rrrrrrrrrs rD_init_core_staters 1  0 9 9 9 M'$C(8(C'''    -5 .7 7 7  <      0N3C3NE+?~/?/?E" $$ Rc%:O6P6P R R    2B773>  $3z||E/E,E(-EEN-//E-//E-< E  EIK#,KE)*.GEM"$FEL LrFcfg}||_g}||_g}||_d|_d|_d|_|S)NT)_root_pre_forward_handles_pre_forward_handles_post_forward_handles_sync_gradients _comm_hook_comm_hook_state)r,rrrs rD_init_runtime_statersK8:&?E#24!5E35"7E EE!E LrFbackward_prefetchforward_prefetchc"||_||_|SrY)rr)r,rrs rD_init_prefetching_staters 0E-E LrFctj|}|r%||jkrt|j|_nd|_|SrY)r get_root_meshr8r%rM_fsdp_extension)r,r0 root_meshs rD_init_extensionrsP -k::I%yE$666 1%2F G G!% LrFctj|_t}t |_||_i}||_|SrY)r#FULL_STATE_DICT_state_dict_typerr_optim_state_dict_config_state_dict_config_unshard_params_ctx)r,state_dict_configunshard_params_ctxs rD_init_state_dict_statersB*:E)<)>)>%=%?%?E"0E57 2E LrFrc|D]U}t|jdkr;d}|D] \}}||ur|}n|sJtd|dVdS)z Verify if the parameters are accepted by FSDP. The only restriction now is that the parameter cannot be a scalar tensor (param.shape == []). rrz/FSDP doesn't support scalar parameters. Change z& to a 1D tensor with numel equal to 1.N)rashapenamed_parametersr5)rrr param_namenameparam_s rD_verify_managed_paramsrs    u{  q J & 7 7 9 9   fF??!%JE#  :M$MMM  !  rFfully_sharded_module param_init_fnsync_module_statesct|j|t|jj}t |jj\}}|s|r|t||jn:|rt||jjn|rtj |fddjD}t|j||t|j|jj_ tt|j} t!|| |r:t#|| jjt(vrt#|| jt-| |S)zHInitialize a ``FlatParamHandle`` from a module ``fully_sharded_module``.Nc8t|duo|jvSrY)rr) submoduler,s rDz0_init_param_handle_from_module..Ls('=i'H'HD'P(8!77rF)check_fnc@h|]}|D]}|Srmbuffers)r]ignored_modulers rDrz1_init_param_handle_from_module..PsK $,,..  rF)_check_single_device_moduler_get_device_from_device_idr:rM_need_to_materialize_moduler_materialize_with_param_init_fn_materialize_meta_moduler)materialize_module_move_module_to_device_get_compute_devicecompute_devicerrr_sync_module_params_and_buffersr-r.r6r=_init_param_handle_from_params) r,rrrrdevice_from_device_idis_meta_moduleis_torchdistX_deferred_initignored_buffersmanaged_paramss ` rD_init_param_handle_from_moduler-s  4e6KYWWW65:u33Ne3U5K33/N/  5 =;T' -1G          "      % ( 8888    #4O    /    E*+?AVWWXXN/@@@' .%2E     "&@ @ @ +$ne6J   #5.:NOOO LrFc 2t|dkrdSt|||jt|j|jj|jj|jj |jj |j |j |j  }||jrJ|j|j||_||j|j<t+jd}|jjr'|jj|kr||dSdSdS)Nr)fsdp_extensionr)rarrSHARDING_STRATEGY_MAPr.roffload_paramsr param_dtype reduce_dtypekeep_low_precision_gradsr-rrshardrrappend flat_paramr_fully_sharded_modulerr flat_param_to)r,rrhandle cpu_devices rDrrrs  6{{a  e56 ( ) * 6  ,   F LLNNN}  L)***EMJPE)&*FGe$$J ')F,=,D ,R,RZ((((()),R,RrF root_modulercd} |t|n t}n5#t$r(}t|dt|z|d}~wwxYw|D]a}t|tjjs"t|dt|zt|rtdb| D]+}tj |s| |,d|D}||vrtjd|| D]?}t|}|,t|dsJ||j@|S) ah Check that ``_ignored_modules`` is an iterable of ``nn.Module`` s without any FSDP instances. Return the modules contained in their module subtrees as a :class:`set`. Nested FSDP instances are excluded, but their already-computed ignored modules are included. ``_ignored_modules`` represents the argument passed by the user to FSDP. z>`ignored_modules` should be an iterable of `torch.nn.Module`s Nzbut got zbut got an iterable with z1`ignored_modules` should not include FSDP modulescth|]5}|D]}t|tj|6Srm)modulesrZ fsdp_fileFullyShardedDataParallel)r]rchilds rDrz'_get_ignored_modules..s\ ^^%% %!CDD  rFzTrying to ignore the top-level module passed into the FSDP constructor itself will result in all parameters being ignored and is not well-supported: r)set TypeErrorrPrZrrrrr5r0traversal_utils _composableaddrrhasattrupdater) r-r msg_prefixignored_root_moduleserrroptional_fsdp_states rDrrsRJQ%5%AC ! ! !suu  QQQ %H5E0F0F%H%HHIIqPQ&RR&%(/22 UJ)ST&\\)S)SSTT T !& ) ) RPQQ Q R %%''--*622 - $ $V , , , *O o%%  ;28 ; ;   !((**II 4Y??  *.0BCC C CC  " "#6#G H H H s$ A#AArcRt}d|D}|||!d|D}|||D]?}t|}|,t |dsJ||j@|S)z Return the parameters of the modules in ``ignored_modules`` and the parameters in ``ignored_parameters``. :class:`FlatParameter` s are excluded from the result. c^h|]*}|D]}t||+Srm) parametersr)r]mps rDrz&_get_ignored_params..sS!!!ALLNN!!'(BTUVBWBW! !!!!rFNc0h|]}t||Srm)r)r]rCs rDrz&_get_ignored_params..s6( ( ( 1CA1F1F( ( ( ( rFr)r4r:r0rr9r)r-rrall_ignored_paramsparams_in_ignored_modulesparams_in_ignored_parametersrr>s rDrrs36%%!!"!!!7888%( ( )( ( ( $ !!">???!((**KK 4Y??  *.0ABB B BB  % %&9&I J J J rFcJt}d|D|fd|D|D]?}t |}|,t |dsJ||j@|S)z6Return the cleaned buffer FQNs in ``ignored_modules``.c@h|]}|D]}|Srmr )r]rBrs rDrz,_get_ignored_buffer_names..sA"""aiikk"",2""""rFc<h|]\}}|v t|Srmr)r]rrbuffers_in_ignored_moduless rDrz,_get_ignored_buffer_names..s;   # V333 k * *333rFNr)r4r:rr0rr9r)r-rall_ignored_buffer_namesrr>rLs @rDrrs *-""'"""##    '2'@'@'B'B   !((**WW 4Y??  *.0GHH H HH $ + +,?,U V V V ##rFc>d|DS)zrReturn the fully prefixed names of all buffers in the module hierarchy rooted at ``root_module`` as a class:`set`.c2h|]\}}t|SrmrK)r]rrhs rDrz$_get_buffer_names.. s2   +9;+&&   rF)r)r-s rDrr s/  =H=V=V=X=X   rFcdt||D}t|dkr)tjd|vr|t ddSt|dkrt d|dS)z Raise an error if ``module`` has original parameters on multiple devices, ignoring the parameters in ``ignored_params``. Thus, after this method, the module must be either fully on the CPU or fully on a non-CPU device. ch|] }|j Srmrr]rs rDrz._check_single_device_module..sRRRu|RRRrFrWrNzTTo support a module with both CPU and GPU params, please pass in device_id argument.rHz;FSDP only supports single device modules but got params on )rrarrr)rrrdevicess rDrrsSR)9&.)Q)QRRRG 7||qU\%00G;;  5    W   S' S S     rFr: device_handlec N|dSt|tjr|ntj|}|jdkre|j^t jd|d|d|d|jd tj|}|S)z Return a ``torch.device`` for the specified ``device_id``. Processes ``device_id`` and returns either the corresponding device or ``None`` if ``device_id`` is ``None``. Nrz"FSDP got the argument `device_id` z on rank zJ, which does not have an explicit index. FSDP will use the current device z6. If this is incorrect, please explicitly call `torch.zk.set_device()` before FSDP initialization or pass in the explicit device index as the `device_id` argument.)rZrrrPindexrrcurrent_device)rr:rUrs rDrr.st 5<88U el9>U>U {e 4  1 1 1 1 10=0L0L0N0N 1 1DJ; 1 1 1   m::<<== MrFc:tt||}td|D}|D]*}||vr|dD] }||jz} +| ot otd|D}||fS)z Return if ``module`` has parameters on meta device and if ``module`` is using torchdistX deferred initialization. At most of the returned bools can be ``True``. If either is ``True``, then ``module`` needs to be materialized. c3$K|] }|jV dSrY)is_metarSs rDr_z._need_to_materialize_module..Xs$CC5CCCCCCrFFrecursec3>K|]}tj|VdSrY)r*is_fakerSs rDr_z._need_to_materialize_module..ds,@@ U##@@@@@@rF)rranyr0r r[_TORCHDISTX_AVAIL)rrrrrrbufrs rDrrKs*6>BBCCNCCNCCCCCN^^%%**  ' ' $$U$33 * *C ck )NN *  A  A @@@@@ @ @ 6 66rFct|s"td|dt|t||}|D] }||dS)Nz Expected z to be callable but got )callabler5rP_get_modules_to_materialize)r-rrmodules_to_materializers rDrris M " "  T T TtM?R?R T T   9oVV( frFrc |p%tj|}t||}d} tj5|D]}t j|d|d}tt|dk}|r+| |d|  ddddS#1swxYwYdS#t$r<} tjdt!| dt#|d| d} ~ wwxYw)NFr\r)rr]zIUnable to call `reset_parameters()` for module on meta device with error z(. Please ensure that your module oftype z* implements a `reset_parameters()` method.)rrrXreno_grad itertoolschainrAr rarto_emptyreset_parameters BaseExceptionrrrrP) r-rrrUmaterialization_devicerfrmodule_state_iterhas_module_statesr=s rDrrws3el$$&&779oVV F]__ . .0 . .%.O%%e%44fnnUn6S6S%%!%(->(?(?$@$@1$D!$.OO+A5OQQQ++--- . . . . . . . . . . . . . . . . . . .   M!$Q M MLL M M M    s<C:BC- C:-C11C:4C15C:: E7D;;EcDg}tj|g}|h}|r|}|||D]C}||vr=t |.||vr*||||D||SrY) collectionsdequepopleftr'childrenrr8)r-rrfqueuevisited_modulesr child_modules rDreres /1  {m , ,E'2mO  +%%f---"OO-- + +LO33*<88@ 77##L111 \***  + "!rFrc tjd |#tj}||g}g}|r|}| fd|dD| fd|dD| D]1}t|tj s||2|ƈfd|D} fd|D} t| | |dStt|d} | | j krt!dSdSdS) a Move ``module`` depending on ``device_from_device_id`` and its current device. This includes moving ignored modules' parameters. - If ``device_from_device_id`` is not ``None``, then this moves ``module`` to the device. - If ``device_from_device_id`` is ``None``, then this does not move ``module`` but warns the user if it is on CPU. Precondition: ``_check_single_device_module()``. rNc32K|]}|jk |VdSrYrR)r]rr,s rDr_z)_move_module_to_device..s<<:------rFFr\c32K|]}|jk |VdSrYrR)r]rr,s rDr_z)_move_module_to_device..s<=J......rFcg|]}|v| Srmrm)r]rCrs rDrpz*_move_module_to_device..s#GGGq/F/F!/F/F/FrFcg|]}|v| Srmrm)r]rCrs rDrpz*_move_module_to_device..s#GGGaa.F.F.F.F.FrF)rrrrrsr'rtextendrAr rurZr1r2_move_states_to_devicenextr_warn_cpu_init) rrrrrvrr  curr_modulerparams_to_move bufs_to_moverr,s `` @rDrrs$e$$J(/:.?.A.A V%'&( ,--//K MM(33E3BB    NN)11%1@@    )1133 , , !)Y-OPP,LL+++% ,&HGGGVGGGGGGG7GGG ~|=RSSS !&.994 @ @E U\Z7777rFr c\t|dkrt|dkrdSt|dkr|dj}n t|dkr |dj}tjd}||D]r}tj5|||_|j$|j||j_dddn #1swxYwYs|D]}|||_dS||krtdSdS)z Move states to the specified device. Precondition: ``_check_single_device_module()`` and module's parameters and buffers have been materialized if needed. rNr)rarrrhtodatagradr)rr rrXr,rrs rDrrs 6{{aCLLA-- 6{{Q) W   *e$$J( K KE K K"XX&;<< :)&+jmm4I&J&JEJO K K K K K K K K K K K K K K K ; ;F ))$9::FKK ; ; : % % & %sAC))C- 0C- c.tjddS)NamThe passed-in `module` is on CPU and will thus have FSDP's sharding initialization run on CPU, which may be slower than on GPU. We recommend passing in the `device_id` argument for FSDP to move `module` to GPU for the sharding initialization. `module` must also be on GPU device to work with the `sync_module_states=True` flag since that requires GPU communication.)rrrmrFrDrrs% M 1rFctt||d}||jjdkr|j}n&t j|}|||krt d|d|d||S)a) Determine and return this FSDP instance's compute device. If the module is already on a non-CPU device, then the compute device is that non-CPU device. If the module is on CPU, then the compute device is the current device. Since this method should be called after materializing the module, any non-CPU device should not be meta device. For now, the compute device is always a CUDA or CUDA-like device with its explicit index. Precondition: ``_check_single_device_module()`` and ``_move_module_to_device()``. Nrz4Inconsistent compute device and `device_id` on rank z: z vs )rrrrPrrXr5)rrrr:rUrrs rDrrs* !&.994 @ @E U\.%77m&B&B&D&DEE(^?T-T-T ;4 ; ; ; ;#8 ; ;    rFc g}|D]}t|tdst|td| t r; \}} fd|D}||| |D]u}| t r; \}} fd|D} || `| vt|t||tddS)z Synchronize module states (i.e. parameters ``params`` and all not-yet-synced buffers) by broadcasting from rank 0 to all ranks. Precondition: ``sync_module_states == True`` and ``self.process_group`` has been set. FTc0g|]}t|Srmgetattr)r]attrdetached_buffers rDrpz3_sync_module_params_and_buffers..Is# R R RD$!?!? R R RrFc0g|]}t|Srmr)r]rdetached_params rDrpz3_sync_module_params_and_buffers..Rs#LLLdGND99LLLrFr)srcN) r r FSDP_SYNCEDsetattrdetachr'__tensor_flatten__r~r'+_check_module_states_for_sync_module_statesr&PARAM_BROADCAST_BUCKET_SIZE) rrr- module_statesrattrsrh inner_buffersr inner_paramsrrs @@rDrr4s)+M.."" 6 6v{E22 6 FK . . .$mmooO,_== 6+==??q R R R RE R R R $$]3333$$_55511 ( 8 8 1%88::HE1LLLLeLLLL   . . . .   0 0 0 0/ >>># rFrc^|r(td|DrtddSdS)Nc3PK|]!}|jtjdkV"dS)rN)rr)r]tensors rDr_z>_check_module_states_for_sync_module_states..csD17 e,,,rFzThe module has CPU parameters or buffers when `sync_module_states=True`, which requires them to be on GPU. Please specify the `device_id` argument or move the module to GPU before passing it to FSDP.)r`r5)rs rDrr`sg ;H  C       rFc#K|} t|}||vrt|s|V'#t$rYdSwxYw)aD Return an iterator over the original parameters in ``module``. The iterator does not return the parameters in ``ignored_params``, any ``FlatParameter`` s (which may be present due to nested FSDP wrapping), or any original parameters already flattened (only relevant when ``use_orig_params=True``). TN)rArr StopIteration)rr param_genrs rDrrmsy!!##I  OOEN**3Ee3L3L*        s(A A Ac t|D]G\}}||vr>t|s/td|d|d|jHdS)a5 Check that original parameters in ``fsdp_module`` have been flattened. The flattened parameters are made invisible to ``named_parameters()`` for the module hierarchy rooted at ``fsdp_module``. This should be called as a sanity check after flattening the wrapped module's parameters. z Found an unflattened parameter: z;  N)rrrr; __class__) fsdp_modulerrrs rD_check_orig_params_flattenedrs?{KK E  & &/A%/H/H &4:44::<<44"'/44 rFcR|tjkr tjn tjSrY)r!rr allreduce_hookreduce_scatter_hook)r.s rD_get_default_comm_hookrs)  0 9 9 9 $$  .rFc,tj|S)NrI)r r>rIs rDrQrQs  %M B B BBrFrY)rrrirrcollections.abcrrrtypingrrrr r r rtorch.distributed distributedr[(torch.distributed.fsdp._exec_order_utilsfsdp_exec_order_utilsr'torch.distributed.fsdp._traversal_utils_traversal_utilsr62torch.distributed.fsdp.fully_sharded_data_parallelfully_sharded_data_parallelr1torch.nnr(torch.distributed.algorithms._comm_hooksr torch.distributed.device_meshr r"torch.distributed.distributed_c10dr$torch.distributed.fsdp._common_utilsrrrrrrr"torch.distributed.fsdp._flat_paramrrrr%torch.distributed.fsdp._limiter_utilsrtorch.distributed.fsdp.apirrrrr r!r"r#torch.distributed.fsdp.wrapr$&torch.distributed.tensor.parallel.fsdpr%torch.distributed.utilsr&torch.utils._python_dispatchr'torch.utils.hooksr(ra torchdistxr)r* ImportErrorintrrr`r\HybridShardProcessGroupTypeProcessGroupTyperr SHARD_GRAD_OP HYBRID_SHARD_HYBRID_SHARD_ZERO2r!r6#NO_RESHARD_AFTER_FORWARD_STRATEGIESrEr7boolrOrJrir~rLrrrrrr4rrrrrrrrrrrrrrrrrrrrrreTensorrrrrrrrrrr>rQrmrFrDrs 9999999999OOOOOOOOOOOOOOOO CCCCCCCCCCCCAAAAAAAAAAAAFFFFFFFFFFFFBBBBBBEEEEEEEEAAAAAA BAAAAA                    0/////DDDDDD<<<<<<FFFFFF2111111........."c"344 #D$5t7H$HIE$"35P"PQR 5>!7!B"$:$H!#9#F(*@*T !( "('# )- 00 0#0(0 W  0 *% 0  0000f( (#(( ((((V#$IJI4IIII  AR    !+!! !!!!H+ 4 d/ /0& ++ + I+hux78+%(,-.%(/9R0SS +++++\I9= :- - I- %-c5</01 -  ----`  I"> > 01>n->*% >  >  >!> >>>>>B           '        : J *    *29d2<6HT&A A)Ac5</01AHbi[$%678 A  A  AAAAH) )  ))))))<66x896 ^6666xBF)!%(*   D$$)$ X$$$$:29S I  % c5</01      <c5</01 %el :7 I7 %7^7 4: 7777<  RYK-. ^       #EL1 ^ %    F""-0^" ")_"""",3 I3 %3&3$EL1 3  3333l   %, $EL1 @ I %$EL1   %   \ F) I)  )$) ))))X   %         I  % bl    , % (.>C$CCCCCCCsCC! C!