P/j >UddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z m Z m Z ddlZddlmZddlmZddgZes:ddlZGd d Zd Zeej.d _eej.d _ydd lmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%ejLe'Z(e r ddl)m*Z*GddejZZ.e.Z/e.e0d<dde1fdZ2GddZddde1de e3dfde e e1dfdefdZy#e+$re(jYdYgwxYw)N)reduce)chain)DictListOptionalTuple TYPE_CHECKINGUnion) is_available)not_noneinit_device_mesh DeviceMeshc eZdZy)_DeviceMeshStubN)__name__ __module__ __qualname__`/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/torch/distributed/device_mesh.pyrrs rrcyNrrrr_init_device_mesh_stubrs rztorch.distributed.device_mesh) _find_pg_by_ranks_and_tag_get_default_group_get_group_tag get_backendget_process_group_ranksget_rankget_world_sizeinit_process_groupis_initialized new_group ProcessGroup) ArrayLikezCDeviceMesh requires numpy >= 1.21 to be installed for type checkingc >eZdZddZddZdddeedfd eeedfddfd Z dddd e eddfd Z dd Z ddde efdZ ededefdZededefdZddd edefdZ ddedede ej&ddfdZdeeedffdZddd ededfdZy)_MeshEnvreturnNcJg|_i|_i|_i|_i|_yr) mesh_stackchild_to_root_mappingmesh_dim_group_optionsroot_to_flatten_mappingflatten_name_to_root_dimsselfs r__init__z_MeshEnv.__init__As702DOGID &  'UWD (  *rrcft|jdk(r td|jdS)Nrz#No device mesh is currently active!)lenr* RuntimeErrorr/s rget_current_meshz_MeshEnv.get_current_meshMs.4??#q("#HII??2& &r device_meshsubmesh_dim_names. submesh_dimsc|Dcgc]=}t|dkDrtfd|njj|d?}}j}g}g}d} t ||D]\} } t| dkDrv|j | d| z | d| z }|j | d| z | t| dz z } |j |j| jd|j | d| z |j j| dtt|j} |D]} | j| |jg| |jdg|}j}|D]"}t!j"||d}||vs!|}$|_|j$|<|Scc}w)Ncrjj|jj|zSr)meshsize)xyr7s rz*_MeshEnv.create_sub_mesh..^s/!1!1!6!6q!9Kzipflattenappendr-_dim_group_infoslistrangendimremovepermutereshaperr device_typer+)r0r7r8r9mesh_dimslice_dim_size mesh_tensor slice_dim_idxslice_dim_group_infonum_dims_flattenmesh_dim_indices mesh_dim_namemesh_dims_remained_idxidxpg_ranks_by_dimcur_rankmesh_ndsubmesh res_submeshs ` rcreate_sub_meshz_MeshEnv.create_sub_meshRs\"!-x=1$ T !%%**8A;7 8N&**KM#%  ! 36|EV3W / -'(1,"-"5"5"21"58H"H 0 47G G#6#K"(()9!)rLrMrNrOrPrrrQr+tuple)r0r7rY root_meshflattened_mesh_dim_nameflatten_dims_in_rootdiminvalid_dim_namesflattened_mesh_dim_sizeremained_dims_in_rootflatten_dim_in_rootr\r]r^flattened_meshres_flattened_meshs rcreate_flatten_meshz_MeshEnv.create_flatten_meshs(55kBI08 8R8R/S$+112889PQ$ $ ! #$8!!9!9:3?!   * * 5 5i D %!hy7789!// :??A!  11"$o%H SUVKL]K^_BCT999!T%A%A)%LL33I>}MM&*ii 0@0@0E0E0G&H #$(y~~/B/B)C$D !'; B#%,,-@A B5inn44&)=gb12 !))+H* 8!+))$1#3" w&)7& 8>GD & &'9 :TfD ( ( 3 3Ir B= QGLMaGbD * *9 5m D% %w$s +I+IcF|jj|d}|s|S|Sr)r+get)r0r7ros rrgz_MeshEnv.get_root_meshs)2266{DII&/; >Y >rc|j|}|j}|r.|r,t|dk(sJd|d}|j||Sy)z Returns the index of the mesh dim in the root mesh. The device_mesh passed in needs to be sliced out from the root mesh or submesh of the root mesh. r;z"The submesh can only be a 1D mesh.rN)rgrEr4get_mesh_dim_by_name)r0r7rochild_mesh_dim_nameschild_mesh_dim_names rget_root_mesh_dimz_MeshEnv.get_root_mesh_dimsf **;7I#.#=#= 1,-28782&:1&=#00$>>&}o5FGD[E_E_D`aK66<<]KL Lrrrbackend pg_optionsc&||f|j|<yr)r,)r0rrrrs r_set_mesh_dim_group_optionsz$_MeshEnv._set_mesh_dim_group_optionss 18/DD ' ' ,rc ||j|k7r td|jj|i|j|}g|j| t fd|Dst d|d dd}g}|D]k}||vr||}|d}|j|n-|jj|}|j|f||krt d|dd|d d |}m|S) z Validate whether the mesh_dim_names is valid for slicing the given device_mesh. If valid, return dim indexes of the slice mesh in the device mesh. z'Cannot create a submesh from a submesh.c3&K|]}|v ywrr).0rYvalid_mesh_dim_namess r z0_MeshEnv._get_slice_mesh_dims..1s!!55szInvalid mesh_dim_names z% specified. Valid mesh_dim_names are .r3z specified. z!Found mesh dim indices to slice: rdz.Mesh dim indices should be in ascending order.) rgr5r.rjrEallrrIrh) r0r7rEr.curr_idxslice_mesh_dimsrY mesh_indicesnext_idxrs @r_get_slice_mesh_dimsz_MeshEnv._get_slice_mesh_dimss[d00=="#LMM  * * 5 5k2 F(,(F(F{(S %$++$*$ %3-n-=>00D/EQHH O!/ $  $==#<]#KL ,B/H#**<8*99?? NH#**H;7x'"1.1AN;O;LBOH $! $$# "rcr|j||}|jjd|jd|jj |}|j }g}|D]H}t |j||fd}||vr|j|gng|_|j|J|S)z` Return all the submeshes of a given mesh dimension of the device mesh. r3FrD) r}r=swapdimsrPr>rrrQrJrI) r0r7rYrRr\r] res_submeshesmesh_1dr_s r_get_all_submeshesz_MeshEnv._get_all_submeshesRs 00mLH)..77HEMMK$$))(3O#++-HM* .$++$1#3"'  7*!11(;<( $$W- .! rr(Nr(rr)r7rr(r)rrrr1r6rstrrintrarryrgr staticmethodrrr}r$Optionsrrrrrrr'r'@s  ' N %N  %S#XN uS#X/ N   N bMQ@ &+@ &  E E E!!5!56  E   E2 # %S/ "2 #h !+ !>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) rQr=.rENTrDr%rFr(c||_t|tjr'|jj dk7rt d|t|tjr.|jjtjn%tj|dtj|_ |r t|nd|_ t|jjj|_d|_|dk7r|r |j%|j't)r&t+dk(rt-j.|_|jt1k(j3}|j5ddvsJ|j5ddkDr|djnd|_yy) Ncpuz!`mesh` must be a CPU tensor, got dtypedevicerxlathreadedr)rr;)rQ isinstancerTensorrtype ValueErrordetachtortensorr=rnrErHtolist_flatten_mesh_list _thread_id_get_or_create_default_group_init_process_groupsr"r threading get_identrnonzeror>_coordinate_on_dim)r0rQr=rErF rank_coordss rr1zDeviceMesh.__init__sq +D $ -$++2B2Be2K #DTF!KLLdELL1   uyy 1\\$uEIIF I &R%,k2F1GqIYIYHZZce((6J)JK%' 'rc g}|jjdk(r|jjtk(rt }t t t}tjjrt|dk(r td|n|}|jt|||jf||_yt |jjD] }|jjd|j!d|jj#|}|D]}|j%}|t&j(vrt&j(|\} } nd\} } t|| | }|j+|vsbt-||kDr t/d|j+d |d |jtt1|||jf||_y) Nr;gloozcpu:gloo,cuda:nccl)rranksr3)NN)rrrzFEach device mesh dimension should get only one process group, but got z in !)r=rMrr rrKrLrcudar rr#rIr group_namerrPr>rrfr,rr4r5r rJ) r0dim_group_infos default_groupr dim_grouprrr\dim_meshsubgroup_ranksrrs rrzDeviceMesh._init_process_groupssACOyy~~"tyy'8N#345zz..0#M2f<&:%H'   &&&y1!,,j%4D !Y!0+C'+ii&8&8S&A&I&IDIINN3/'O %4#)1):/"H"HH!0 F Fs K ' *3=/GZ %."0$+'1%  ==?n<"?3c9&2&lmqmzmzm|l}~**8)9%<'"!",22$28I3F$G$2$-$8$8!";#+X%4D !rcDtjj||Sr)rfr*rIr/s r __enter__zDeviceMesh.__enter__8s  & & - -d 3Krc@tjjyr)rfr*pop)r0exc_type exc_value exc_tracebacks r__exit__zDeviceMesh.__exit__>s  & & * * ,rc|js-d|jd|jjd}|Sd|jd|jjd|jd}|S)Nz DeviceMesh('z', )z, mesh_dim_names=)rErQr=r)r0device_mesh_reprs r__repr__zDeviceMesh.__repr__Bs**t//0DII4D4D4F3GqI  $ #$D$4$4#5S9I9I9K8LL]^b^q^q]rrst  $ #rct|dd|_|jsQt|j|jj |j |j|jf|_|jS)N_hash) rrhashrr=shaperQrErr/s r__hash__zDeviceMesh.__hash__Jsc w5DJ::!// ((++  :: rotherct|tsyt|t|k(ry|j|jk(xr~|jj |jj k(xrO|j |j k(xr4|j|jk(xr|j|jk(S)NFT) rridrr=rrQrEr)r0rs r__eq__zDeviceMesh.__eq__YseZ0$x2e9$++u/G/GG< 5::+;+;;<((E,=,==<++u/C/CC<5+;+;; rc|js tdt|tr|fn|}||jk(r|Stj ||}tj |||}|S)aU Slice the current DeviceMesh based on the mesh_dim_names given to create a submesh. The submesh created consists of the dimensions and the communicators indicated by ``mesh_dim_names`` Args: mesh_dim_names (Union[str, Tuple[str]]): the name or the tuple of names of the mesh dimension of the DeviceMesh to create the submesh for. Returns: A :class:`DeviceMesh` object The following program runs on each process/rank in an SPMD manner in a world size of 8. In the first example: Calling mesh_2d["tp"] on rank 0, 1, 2, 3 returns a 1D submesh of DeviceMesh:([0, 1, 2, 3]). Calling mesh_2d["tp"] on rank 4, 5, 6, 7 returns a 1D submesh of DeviceMesh:([4, 5, 6, 7]). Calling mesh_2d["dp"] on rank 0, 4 returns a 1D submesh of DeviceMesh:([0, 4]). Calling mesh_2d["dp"] on rank 1, 5 returns a 1D submesh of DeviceMesh:([1, 5]). Calling mesh_2d["dp"] on rank 2, 6 returns a 1D submesh of DeviceMesh:([2, 6]). Calling mesh_2d["dp"] on rank 3, 7 returns a 1D submesh of DeviceMesh:([3, 7]). In the second example: Calling mesh_3d["dp", "cp"] on rank 0, 1, 4, 5 returns a 2D submesh of DeviceMesh:([[0, 1], [4, 5]]). Calling mesh_3d["dp", "cp"] on rank 2, 3, 6, 7 returns a 2D submesh of DeviceMesh:([[2, 3], [6, 7]]). Calling mesh_3d["cp", "dp"] on rank 0, 1, 4, 5 returns a 2D submesh of DeviceMesh:([[0, 4], [1, 5]]). Calling mesh_3d["cp", "dp"] on rank 2, 3, 6, 7 returns a 2D submesh of DeviceMesh:([[2, 6], [3, 7]]). Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize a 2D device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh_2d = init_device_mesh(device_type="cuda", (2,4), mesh_dim_names=("dp", "tp")) >>> tp_mesh = mesh_2d["tp"] >>> dp_mesh = mesh_2d["dp"] >>> >>> # Initialize a 3D mesh. >>> mesh_3d = init_device_mesh(device_type="cuda", (2,2,2), mesh_dim_names=("dp", "pp", "cp")) >>> # The order of the mesh_dim_names provided deteremines the order of dimensions in the submesh. >>> dp_cp_mesh = mesh_3d["dp", "cp"] >>> cp_dp_mesh = mesh_3d["cp", "dp"] z1Cannot slice a DeviceMesh without mesh_dim_names!)rEr5rrrfrra)r0rErr_s r __getitem__zDeviceMesh.__getitem__gs|Z&&"#VWW&0%D!. !4!44 "1"F"F.#*99./rrRct|ds td|jjdkDr'|%td|jjddd|jjdk(r#|!t t |j d dd Stj|}tjj|d}|r8||jvr&||j d dd }t t |St|trtj||n|}t t |j |dd S) a Returns the single ProcessGroup specified by mesh_dim, or, if mesh_dim is not specified and the DeviceMesh is 1-dimensional, returns the only ProcessGroup in the mesh. Args: mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index of the mesh dimension. Default is None. Returns: A :class:`ProcessGroup` object. rJz*DeviceMesh process groups not initialized!r;NFound the DeviceMesh have dimensionsJOptional kwarg `mesh_dim` needs to be specified when device_mesh.ndim > 1.zmIf you want to get the list of all the ProcessGroups in the DeviceMesh,please use `get_all_groups()` instead.r)hasattrr5r=rMr rrJrfrgr-r{rkrrr})r0rRror-rs r get_groupzDeviceMesh.get_groups\4!34"#OPPyy~~!h&6"00@ L`=yy~~"x'7-t/D/DQ/G/KL(55d;I&5&M&M&Q&Q4' #'87N7S7S7U+U"9("C"T"TUV"WXZYZ"[ 9? KLL"(C0$88xH!  -t/D/DX/NrPQ/RSrct|jjDcgc]}|j|c}Scc}w)z Returns a list of ProcessGroups for all mesh dimensions. Returns: A list of :class:`ProcessGroup` object. )rLr=rMr)r0is rget_all_groupszDeviceMesh.get_all_groupss.05TYY^^/DE!DNN1%E EEs=regroupct|trt|}t|tjr|j |k7s|||k7rt dt|d|tj|dtj}t|||d}t|||jfg|_ |St|}t|dk(r t d| t d t|tjr/|j!j#tjd n%tj|dtj}|j$t|k7r)t d |j d t|d t|||d}|Dcgc]#}t|t||jf%c}|_ |Scc}w)aM Constructs a :class:`DeviceMesh` with ``device_type`` from an existing :class:`ProcessGroup`. The constructed device mesh has number of dimensions equal to the number of groups passed. If more than one group is passed, then the ``mesh`` argument is required. z Invalid mesh z for ProcessGroup with ranks rrFrDrz.Expects at least one ProcessGroup to be passedz0Must pass mesh if passing multiple ProcessGroups)rrzEExpects mesh with ndim equal to number of ProcessGroups but got mesh z and z ProcessGroups)rr$rrrrrrrrrrrrJrKr4rrrM)rrQr=rE group_ranksr7groupss r from_groupzDeviceMesh.from_groups %.5e< tU\\2t{{} 7S&4;+>$'D {2OP[}]||KUYYO(#1"'  $E*K9I9IJ0 ,#"%[F6{a !QRR| !STTdELL1   uyy ?\\$uEIIF  yyCK'  KKM?%F }NL%T.PUK$ ,  #5)+E2$$,K ( ,s3(G%cp||jjS|jj|Sr)r=rr>)r0rRs rr>zDeviceMesh.sizes*(0(8499??$ VdiinnX>V Vrc.|jjSr)r=rMr/s rrMzDeviceMesh.ndims99>> !rc@t|jjSr)rnr=rr/s rrzDeviceMesh.shape#s) )rctS)z: Returns the current global rank. )rr/s rrzDeviceMesh.get_rank's : rc|jdkDr&|$td|jjdd|d}t|j |}t |t sJdtt|S)a{ Returns the local rank of the given mesh_dim of the DeviceMesh. Args: mesh_dim (str/int, optional): it can be the name of the mesh dimension or the index of the mesh dimension. Default is None. Returns: An integer denotes the local rank. The following program runs on each process/rank in an SPMD manner. In this example, we have 2 hosts with 4 GPUs each. Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 0, 1, 2, 3 would return 0. Calling mesh_2d.get_local_rank(mesh_dim=0) on rank 4, 5, 6, 7 would return 1. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 0, 4 would return 0. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 1, 5 would return 1. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 2, 6 would return 2. Calling mesh_2d.get_local_rank(mesh_dim=1) on rank 3, 7 would return 3. Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import DeviceMesh >>> >>> # Initialize device mesh as (2, 4) to represent the topology >>> # of cross-host(dim 0), and within-host (dim 1). >>> mesh = DeviceMesh(device_type="cuda", mesh=[[0, 1, 2, 3],[4, 5, 6, 7]]) r;rrrrz1We expect ProcessGroup before calling `get_rank`!)rMr5r=r rrr$r)r0rRmesh_dim_groups rget_local_rankzDeviceMesh.get_local_rank-s8yy1}!1"00@ L`!%dnnX&>?N  CB CH^45 5rc6|jr |jSdS)z Return the relative indices of this rank relative to all dimensions of the mesh. If this rank is not part of the mesh, return None. N)rr/s rget_coordinatezDeviceMesh.get_coordinateWs /3.E.E4** O4 OrrYc\|js tdtj||S)a\ Returns a 1D DeviceMesh by flattening the current DeviceMesh. If no mesh_dim_name is provided, the default is a string concatentaing the mesh_dim_names of the given submesh with each mesh_dim_name separated by "_". For example, if we have a 3D mesh DeviceMesh([[[0, 1], [2, 3]], [[4, 5], [6, 7]]], mesh_dim_names=("dp", "cp", "tp")), calling mesh_3d["dp", "cp"]._flatten() will create a 1D submesh DeviceMesh([0, 1, 2, 3], mesh_dim_names=("dp_cp",)) on rank 0, 1, 2, 3 and a 1D submesh DeviceMesh([4, 5, 6, 7], mesh_dim_names=("dp_cp",)) on rank 4, 5, 6, 7. After the flattened dimension is created, to access the flattened dimesnion in mesh_3d, one can use the existing slicing method to obtain the flattened mesh through calling mesh_3d["dp_cp"]. z3Cannot flatten a DeviceMesh without mesh_dim_names!)rEr5rfry)r0rYs r_flattenzDeviceMesh._flatten^s2&&"I#66t]K Krrrr)%rrr__doc__r__annotations__rrrrr boolr1rrrrrrobjectrrrr$rrrrrr>propertyrMrrrrrrrrrrzsV% Nll sCx119="& ( (  k12(  %U38_5 (   ( ( T (:I 4V   - $c $   4 = "'U38_(<"==  = ~, huS#X&?, <, \ FD$6 F @D<  9= < tL'99:< < 5{!:;<<  %U38_5 <   <  < | W# W# W  "# "  "  *5c? *  * c  ( 68E#s(O+D( 6PS( 6T PHT#Y$7 P L(3- L< Lrre mesh_shape.rEr(c "|ktt|t|k7rtdd|t|t|k7r%tddt|dt|d|r |jstd|d d t j d 5t j tj|tj j|}dddt|| }|S#1swYxYw)a Initializes a `DeviceMesh` based on `device_type`, `mesh_shape`, and `mesh_dim_names` parameters. This creates a DeviceMesh with an n-dimensional array layout, where `n` is the length of `mesh_shape`. If `mesh_dim_names` is provided, each dimension is labeled as `mesh_dim_names[i]`. .. note:: `init_device_mesh` follows SPMD programming model, meaning the same PyTorch Python program runs on all processes/ranks in the cluster. Ensure `mesh_shape` (the dimensions of the nD array describing device layout) is identical across all ranks. Inconsistent `mesh_shape` may lead to hanging. .. note:: If no process group is found, init_device_mesh will initialize distributed process group/groups required for distributed communications behind the scene. Args: device_type (str): The device type of the mesh. Currently supports: "cpu", "cuda/cuda-like". Passing in a device type with a GPU index, such as "cuda:0", is not allowed. mesh_shape (Tuple[int]): A tuple defining the dimensions of the multi-dimensional array describing the layout of devices. mesh_dim_names (Tuple[str], optional): A tuple of mesh dimension names to assign to each dimension of the multi-dimensional array describing the layout of devices. Its length must match the length of `mesh_shape`. Each string in `mesh_dim_names` must be unique. Returns: DeviceMesh: A :class:`DeviceMesh` object representing the device layout. Example:: >>> # xdoctest: +SKIP("no rank") >>> from torch.distributed.device_mesh import init_device_mesh >>> >>> mesh_1d = init_device_mesh("cuda", mesh_shape=(8,)) >>> mesh_2d = init_device_mesh("cuda", mesh_shape=(2, 8), mesh_dim_names=("dp", "tp")) Nz"Each mesh_dim_name must be unique.z/Found repeated mesh_dim_name in mesh_dim_names z6mesh_shape and mesh_dim_names should have same length!zFound len(mesh_dim_names): z and len(mesh_shape):rz4Device type with GPU index is not supported but got rdzUIf you maintained a 'torch.device' object, it's recommended to pass in 'device.type'.rr)rQr=rE) r4setr5isalpharrarangerlrmrviewr)rQr rEr=r7s rr r rsR  %3~&'3~+>>"8EnEUV :#n"55"L1#n2E1FF[\_`j\k[llmn {224F{mSUVg \\%  Y<< * 5UYYGLLZXD Y #)  Y Ys %ADD)r)4loggingrlr functoolsr itertoolsrtypingrrrrr r rtorch.distributedr torch.utils._typing_utilsr __all__sysrrmodulesrr "torch.distributed.distributed_c10drrrrrrr r!r"r#r$ getLoggerrlogger numpy.typingr% ImportErrorwarninglocalr'rfrrrrrrrr sR DD *. | ,~   ?NCKK/0;0KK'    W  x (F  . m!9??m!^ !) OX*11vLvLx59 GG#s(OG!sCx1 G  Go  NNU  s.DDD