P/jT5 ddlZddlmZmZmZmZmZddlZddlm Z ddl m cm cm Zddlm cmZddlmZmZmZmZddl mZddlmZddlmZddlmZddlmZdd l m!Z!dd l"m#Z#dd l$m%Z%m&Z&m'Z'mZ(dd l)m*Z*m+Z+d gZ,de&deejZejZffdZ.de&de/deejZejZffdZ0de&deejZejZffdZ1de&de/defdZ2de&dejfdefdZ4de&dejffdZ5dejldejnde/dejlfdZ8dejnde/de/de/de jfdejnf d Z9dejnde/d!e%de&fd"Z:dejndeejneeffd#Z;de&d$ee%dejnfd%Z<Gd&d eZ=y)'N)AnycastListOptionalTuple)Shard ShardedTensorShardedTensorMetadataTensorProperties) ShardMetadata)ChunkShardingSpec)_mesh_resources)_set_fsdp_flattened)FSDPExtensions)_create_chunk_sharded_tensor)_remote_device) DeviceMeshDTensor Replicater)_flatten_tensor_unflatten_tensorDTensorExtensionstensorreturnc|j}|jdk(sJd|jd}dgt|j z}|j d}|jdj r3t t|j}|j ||z}|||<tj||jj fS)N&Only 1D DeviceMeshes currently handledr)mesh_dim) device_meshndim placementslensizeis_shardrDSharddimtorchSize _local_tensor)rr placementoffsets num_chunks shard_dim chunk_sizes i/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/torch/distributed/tensor/parallel/fsdp.py_get_boxr0 s$$K   q J"JJ !!!$IcC &&G!!1!-J $$&+// [[+z9 '  JJw !5!5!:!:!< ==idxcxt|\}}tj|Dcgc]}||z c}|fScc}wN)r0r'r()rr2r+r#vals r/ _get_box_forr60s6V$MGT JJW5cc 5 6 ==5s 7c`|j}|j}|Jt||dS)Nr)rget_coordinater6)rrcoords r/_get_local_boxr:5s8$$K  & & (E    a ))r1dt current_rankc|j}|jdk(sJdt|\}}tt |t |d|d|j j S)Nrrrank:/ shard_offsets shard_sizesr*)rr r:r listr)device)r;r<meshr+sizess r/_create_shard_md_from_dtrG<sg >>D 99>CCC>#B'NGU 7mK,q)9)9)@)@(AB r1dt_pgc  g}tj|}|dkDrdnd}|jdjr|j }nd}t |D]a}t ||\}}|jtt|t|d|dkDr|n|d|jjct||j t|j|j|j S)Nrrr>r?r@)dtypelayout requires_grad)shards_metadatar#tensor_properties)distget_rankr!r$r#ranger6appendr rCr)rDr r rJrKrL) r;rH shards_mdmy_rankscapegoat_rank shard_countir+rFs r/!_create_sharded_tensor_md_from_dtrXHs ImmE"G!A+Q1N }}Q  "jjl  ;   %b!, "7m Ka!eNA2CSCSCZCZB[\      !! WWY*((99**   r1cf|j}|jdk(sJd|jS)Nrr)rr get_group)r;rEs r/ _get_dt_pgr[os. >>D 99>CCC> >> r1specrankct|ts|Sd}|jD]G}tt|}|j |k(s'|j |j k7sEd}n|rtj|}t|jD]o\}}tt|}|j |k(s*|j |j k7sHt d|d|j |j|<q|S)z Rewrite ``spec`` to match the device of ``tensor``. FSDP.sharded_optim_state_dict sneakly ships optimizer state to CPU so if the original ShardingSpec produces CUDA metadata, ST construction bombs. FTr>r?) isinstancer r!rrr]rDcopydeepcopy enumerate)r\rr]rewriteprWr*s r/_rewrite_spec_if_neededreus d- . G __  # 668t  fmm ;G   }}T"%doo6 TLAy^Y7I~~4'I,<,<,>&--,O%3eD66==/4R%S" T Kr1 world_sizenum_devices_per_nodepgc t|turt|jdk(sJ|j }t |||||}|jd}t |tj|jg}tj|j} d| j_ tj|| |jd} | St|tur|j} | j dk(sJd|j"}t |||t$j&j)|}t+|} t |t-|t/j0| g}t3|| } d| j_ tj|| | d} | St |||||S)NrrF)sharded_tensor_metadata process_group init_rrefsr)typer r" local_shards local_tensorrrr`rametadatarNrL+_init_from_local_shards_and_global_metadata_process_grouprrr r)r'cuda device_countr[rGrOrPrX) rr]rfrgrh inner_paraminner_stouter_local_shardshardsst_metast_outerrrHs r/ _chunk_tensorr{s F|}$6&&()Q...))+ /      #//1!4 (DMM*;*D*DE F -- 1227!!/ LL $+ //    f (( 1$N&NN$** /    JJ # # %   6" (4VT]]5=QR S 4FEB27!!/ LL $+  +       r1rctj|}| td|jdkrtd|jdd|j j }t |tjrt |tst|jDcgc] }t}}t|jDcgc] }t}}td|d<tj|||dj|| S|j}|d}|j!}t|jDcgc] }t}}||d <t|jD cgc] } t}} td|d <||d <tj|||dj|| Scc}wcc}wcc}wcc} w) z Shard a tensor to chunks along the first dimension. The local rank will gets its corresponding chunk as the local tensor to create a DTensor. z4No parent device_mesh is found for FSDP device_mesh.z!Found parent device_mesh of ndim=,zbut meshes must be at least 2D.rF) run_checkrr!)r get_root_mesh RuntimeErrorr clonedetachr_r'TensorrrQrr% from_local redistributer!to_local) rr]r root_mesh_replicate_placementsshard_placements tp_placements tp_placementrWs r/_chunk_dtensorrs --k:IQRR~~/ /?q A -  \\^ " " $F &%,,' 670K6;9>>5JK KK16y~~1FGAIKGG$Qi!! I3u ,!'   )) $Q' "6;9>>5JK KK#/R 16y~~1FGAIKGG%ay+!! I3u ,!'   9 LG* LGs+GGG!G&ctt|j}t|dk(r?t |dj tur!|dj }|j}|}|t|dkDr|fSgfS)Nrr)rr rnr"rmr)rrx inner_tensors r/_pre_load_state_dictrsz- ( 5 5 7F 6{aD!1!12mCay'' **, c&kAoF 662 66r1 parent_meshc"||jk(sJttj|j}t dt |dz D]}t||<|j|j|}|jS)zGAll gather a DTensor in its FSDP dimension and return the local tensor.rrr) rrCr`rar!rQr"rrr)rrr!rWs r/_all_gather_dtensorr)s &,, ,, ,dmmF$5$567J1c*o) *$! 1 $  &&!F ?? r1ceZdZdZdfd Zdej deej ee ffdZ dej de dej fdZ ddej de d e d e d e jd eejdej fd Zdej de dedej fdZdej deej eeffdZdedeedej fdZxZS)rz DTensorExtension is the TensorFlattener extension needed for 2D FSDP + TP. This is the implementation for FSDPExtensions defined in https://github.com/pytorch/pytorch/blob/main/torch/distributed/fsdp/_fsdp_extensions.py rct|d|_||_tj j |j|_yr4)super__init__compute_stream device_handler'_dynamodisablepost_unflatten_transform)selfr __class__s r/rzDTensorExtensions.__init__Es= "*). (=(=d>[>[(\%r1rct|Sr4)rrrs r/pre_flatten_transformz'DTensorExtensions.pre_flatten_transformMsv&&r1param_extensionc|jxs|jj}|jj|5t |||j|j}t ||cdddS#1swYyxYw)N)rr)rrcurrent_streamstreamrr)rrrrresults r/rz*DTensorExtensions.post_unflatten_transformSs}$$K(:(:(I(I(K    & &v .  '"00#22 F  '   s 0A>>Br]rfrgrhrDc t|||||Sr4)r{)rrr]rfrgrhrDs r/ chunk_tensorzDTensorExtensions.chunk_tensorfsVT:7KRPPr1rct|||Sr4)r)rrr]rs r/ chunk_dtensorzDTensorExtensions.chunk_dtensorqs fdK88r1ct|Sr4)rrs r/pre_load_state_dict_transformz/DTensorExtensions.pre_load_state_dict_transformys$F++r1rct||Sr4)r)rrrs r/all_gather_dtensorz$DTensorExtensions.all_gather_dtensors #6;77r1)rNr4)__name__ __module__ __qualname____doc__rr'rrrrrrintrO ProcessGrouprDrrrrrrrr __classcell__)rs@r/rr=sW]' ' u||Xc]* +' ll58 4*. Q  Q Q Q " Q    Q& Q  Q9 99 9  9, , u||T%[( ), 88j)8  8r1)>r`typingrrrrrr'torch.distributed distributedrO&torch.distributed._shard.sharding_spec_shard sharding_spec shard_spec"torch.distributed.distributed_c10ddistributed_c10dc10d'torch.distributed._shard.sharded_tensorrr r r r :torch.distributed._shard.sharding_spec.chunk_sharding_specr torch.distributed.device_meshr$torch.distributed.fsdp._common_utilsr'torch.distributed.fsdp._fsdp_extensionsr#torch.distributed.fsdp._shard_utilsrtorch.distributed.remote_devicertorch.distributed.tensorrrrr%6torch.distributed.tensor.parallel._data_parallel_utilsrr__all__r(r0rr6r:rGrrXr[ ShardingSpecrrer{rrrrr1r/rs; 33 ;;11 AX9DBL:TT   >W >uzz5::'=!> > >>s>uUZZ5K/L> *7*uUZZ-C'D*    $$))$$N7t00   ! !+0<<?B:G LLG G G  G  G  \\ G T> LL> > >  > B 7 LL 7 5<<e $% 7 *% \\(G8G8r1