Ë
    P/æji  ã                   óÎ  — d dl mZmZmZmZ d dlmZ d dlZd dlm	Z
 d dlmZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZm Z  d d	l!m"Z"m#Z#m$Z$ d d
l%m&Z& d dl'm(Z( d dl)m*Z*m+Z+m,Z,m-Z- d dl.m/Z/  ee¬«       ede0¬«      ddddddddddddœdejb                  dee
jd                     dee/   dee-   dee,   dee+   deeej                  jb                        deee3ejh                  f      deeejb                  gdf      de5de5deeeej                  jl                        eeej                  jb                        f   dejb                  fd „«       «       Z7y)!é    )ÚCallableÚIterableÚOptionalÚUnion)Ú
deprecatedN)Úcontract)Ú_get_module_stateÚ_insert_module_state)Ú
_FSDPState)Ú_annotate_modules_for_dynamo)
Ú_init_buffer_stateÚ_init_core_stateÚ_init_device_handleÚ_init_ignored_module_statesÚ_init_param_handle_from_moduleÚ_init_prefetching_stateÚ_init_process_group_stateÚ_init_runtime_stateÚ_init_state_dict_stateÚHYBRID_SHARDING_STRATEGIES)Ú_register_post_forward_hookÚ_register_pre_forward_hookÚ_register_root_pre_forward_hook)Ú_register_all_state_dict_hooks)Ú
_auto_wrap)ÚBackwardPrefetchÚ
CPUOffloadÚMixedPrecisionÚShardingStrategy)Ú_Policy)Ú	state_clsa1  `torch.distributed._composable.fully_shard` is being deprecated. You can continue to use the wrapper based FSDP. See usage in: https://github.com/pytorch/pytorch/blob/main/torch/distributed/fsdp/fully_sharded_data_parallel.py. `torch.distributed._composable.fully_shard` will be removed after PyTorch 2.5.)ÚcategoryF)Úprocess_groupÚpolicyÚstrategyÚmixed_precisionÚcpu_offloadÚignored_modulesÚ	device_idÚparam_init_fnÚsync_module_statesÚforward_prefetchÚignored_statesÚmoduler#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   Úreturnc                óÊ  — t         j                  j                  d«       |�t        |t        «      st        d|› �«      ‚t        j                  | «      }t        || ||«      }t        || |j                  |«      }t        | |j                  d«       t        ||||«      }|�X||||||||	|
|dœ
}|t        v r|j                  |j                   f|d<   t#        | ||j                  |j                  |t        «       t%        ||xs t&        j(                  ||dddd¬«      }t+        |«      }t-        |t.        j0                  |
¬«      }t3        || «      }t5        || |||	«      }t7        |«      }t9        |«       t;        || «       t=        || «       t?        || «       tA        | |«       | jC                  «       D ])  }||jD                  v sŒtG        |«      �ŒtA        ||«       Œ+ | S )	zDApplies ``FullyShardedDataParallel`` (FSDP) semantics to ``module``.ztorch.distributed.fully_shardzExpects a `_Policy` but got T)
r#   r%   r&   r'   r(   r)   r*   r+   r,   r-   r#   é   )Úlimit_all_gathersÚuse_orig_paramsÚbackward_prefetch_limitÚforward_prefetch_limit)r,   )$ÚtorchÚ_CÚ_log_api_usage_onceÚ
isinstancer    Ú
ValueErrorÚfully_shardÚstater   r   Ú_ignored_paramsr   Ú_ignored_modulesr   r   r#   Ú_inter_node_pgr   r   r   Ú
FULL_SHARDr   r   r   ÚBACKWARD_PREr   r   r   r   r   r   r   r
   ÚmodulesÚ_fully_sharded_module_to_handler	   )r.   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r<   Úroot_kwargsÚ	submodules                  úl/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/torch/distributed/_composable/fully_shard.pyr;   r;   (   sÿ  € ô4 
‡H�H× Ñ Ð!@ÔAàÐ¤*¨V´WÔ"=ÜÐ7¸°xÐ@ÓAÐAÜ×Ñ˜fÓ%€EÜ'¨¨v°ÈÓW€EÜ  v¨u×/DÑ/DÀiÓP€EÜ  ¨×)?Ñ)?ÀÔFÜ% e¨]¸HÀfÓM€EØÐà*Ø Ø.Ø&Ø.Ø"Ø*Ø"4Ø 0Ø,ñ
ˆð Ô1Ñ1Ø,1×,?Ñ,?À×AUÑAUÐ+VˆK˜Ñ(ÜØØØ×"Ñ"Ø×!Ñ!ØÜô	
ô ØØÒ/Ô$×/Ñ/ØØØØØ !Ø ô	€Eô   Ó&€EÜ#ØÔ×,Ñ,Ð?Oô€Eô ˜u fÓ-€EÜ*Øˆv�y -Ð1Có€Eô # 5Ó)€EÜ" 5Ô)Ü˜u fÔ-Ü  vÔ.Ü# E¨6Ô2ô ˜ Ô'Ø—^‘^Ó%ò 3ˆ	à˜×>Ñ>Ò>Ü! )Ó,Ñ4ä  ¨EÕ2ð3ð €Mó    )8Útypingr   r   r   r   Útyping_extensionsr   r6   Útorch.distributedÚdistributedÚdistÚtorch.nnÚnnÚ&torch.distributed._composable.contractr   Ú#torch.distributed._composable_stater	   r
   Ú$torch.distributed.fsdp._common_utilsr   Ú$torch.distributed.fsdp._dynamo_utilsr   Ú"torch.distributed.fsdp._init_utilsr   r   r   r   r   r   r   r   r   r   Ú%torch.distributed.fsdp._runtime_utilsr   r   r   Ú(torch.distributed.fsdp._state_dict_utilsr   Ú"torch.distributed.fsdp._wrap_utilsr   Útorch.distributed.fsdp.apir   r   r   r   Útorch.distributed.fsdp.wrapr    ÚFutureWarningÚModuleÚProcessGroupÚintÚdeviceÚboolÚ	Parameterr;   © rG   rF   ú<module>ra      sÎ  ðç 6Ó 6Ý (ã Ý  Ý Ý ;ß WÝ ;Ý M÷÷ ÷ ÷ñ õ
 TÝ 9÷ó õ 0ñ 
�JÔÙðUð ôð 26Ø $Ø+/Ø04Ø(,Ø;?Ø48Ø;?Ø$Ø"ð 	òSØ�I‰IðSð ˜D×-Ñ-Ñ.ðSð �WÑð	Sð
 Ð'Ñ(ðSð ˜nÑ-ðSð ˜*Ñ%ðSð ˜h u§x¡x§¡Ñ7Ñ8ðSð ˜˜c 5§<¡<Ð/Ñ0Ñ1ðSð ˜H b§i¡i [°$Ð%6Ñ7Ñ8ðSð ðSð ðSð Ø�˜%Ÿ(™(×,Ñ,Ñ-Ñ.°¸À%Ç(Á(Ç/Á/Ñ9RÑ0SÐSñðSð  ‡Y�Yò!Sóó  ñSrG   