Ë
    P/æjxj  ã                   ó  — d dl Z d dlZd dlZd dlmZ d dlmc mZ d dl	Z
d dlZd dlmZ 	 d&dededefd„Zd'd„Z	 	 	 	 	 d(dededed	ed
edededededefd„Zd„ Zd„ Zd„ Z G d„ dej0                  «      Z G d„ dej0                  «      Z G d„ dej6                  «      Z G d„ dej:                  «      Z G d„ dej0                  «      Z G d„ dej0                  «      Z  G d „ d!ej0                  «      Z!d"„ Z"d#„ Z#d$„ Z$d)d%„Z%y)*é    N)Ú	rearrangeÚlengthÚdimÚshiftc                 óf  — |dz  dk(  sJ ‚|t        j                  | |¬«      j                  ddd«      z   }|dz  }t        j                  |dz  |¬«      j                  ddd«      }||||dz
  z  z  z  }t        j                  t        j                  |«      t        j
                  |«      gd¬«      S )Né   r   ©Údeviceéÿÿÿÿé   ©r   )ÚtorchÚarangeÚviewÚcatÚcosÚsin)	r   r   r   r
   Ú
max_periodÚposÚhalf_dimÚadimÚphases	            úU/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/demucs/transformer.pyÚcreate_sin_embeddingr      s¯   € ð �‰7�aŠ<Ðˆ<Ø
”%—,‘,˜v¨fÔ5×:Ñ:¸2¸qÀ!ÓDÑ
D€CØ�a‰x€HÜ�<‰<˜˜q™¨Ô0×5Ñ5°a¸¸BÓ?€DØ�: $¨(°Q©,Ñ"7Ñ8Ñ9€EÜ�9‰9ä�I‰I�eÓÜ�I‰I�eÓð	
ð ôð ó    c                 óŠ  — | dz  dk7  rt        dj                  | «      «      ‚t        j                  | ||«      }t	        | dz  «      } t        j
                  t        j                  d| d«      t        j                  |«      | z   z  «      }t        j                  d|«      j                  d«      }t        j                  d|«      j                  d«      }t        j                  ||z  «      j                  dd«      j                  d«      j                  d|d«      |d| d…dd…dd…f<   t        j                  ||z  «      j                  dd«      j                  d«      j                  d|d«      |d| d…dd…dd…f<   t        j                  ||z  «      j                  dd«      j                  d«      j                  dd|«      || dd…dd…dd…f<   t        j                  ||z  «      j                  dd«      j                  d«      j                  dd|«      || dz   dd…dd…dd…f<   |ddd…f   j                  |«      S )z¶
    :param d_model: dimension of the model
    :param height: height of the positions
    :param width: width of the positions
    :return: d_model*height*width position matrix
    é   r   zHCannot use sin/cos positional encoding with odd dimension (got dim={:d})r   ç        r   N)Ú
ValueErrorÚformatr   ÚzerosÚintÚexpr   ÚmathÚlogÚ	unsqueezer   Ú	transposeÚrepeatr   Úto)	Úd_modelÚheightÚwidthr
   r   ÚpeÚdiv_termÚpos_wÚpos_hs	            r   Úcreate_2d_sin_embeddingr1   %   s  € ð ��{�aÒÜð+ß+1©6°'«?ó
ð 	
ô 
�‰�W˜f eÓ	,€Bä�'˜A‘+Ó€GÜ�y‰yÜ�‰�S˜' 1Ó%¬$¯(©(°:Ó*>ÀÑ*HÐ(IÑIó€Hô �L‰L˜˜eÓ$×.Ñ.¨qÓ1€EÜ�L‰L˜˜fÓ%×/Ñ/°Ó2€Eä�	‰	�%˜(Ñ"Ó#×-Ñ-¨a°Ó3×=Ñ=¸aÓ@×GÑGÈÈ6ÐSTÓUð €qˆ�€{’A’qÐÑô 	�	‰	�%˜(Ñ"Ó#×-Ñ-¨a°Ó3×=Ñ=¸aÓ@×GÑGÈÈ6ÐSTÓUð €qˆ�€{’A’qÐÑô 	�	‰	�%˜(Ñ"Ó#×-Ñ-¨a°Ó3×=Ñ=¸aÓ@×GÑGÈÈ1ÈeÓTð €w€z�€z’1’aÐÑô 	�	‰	�%˜(Ñ"Ó#×-Ñ-¨a°Ó3×=Ñ=¸aÓ@×GÑGÈÈ1ÈeÓTð €w��{€�Q€šš1ÐÑð ˆd’Aˆg‰;�>‰>˜&Ó!Ð!r   Ú
batch_sizeÚmean_normalizeÚaugmentÚmax_global_shiftÚmax_local_shiftÚ	max_scaler
   r   c
                 óv  — |dz  dk(  sJ ‚dt        j                  | «      j                  ddd«      z  }
|
j                  d|d«      }
|r|
t        j                  |
dd¬«      z  }
|r¹t
        j                  j                  | |­d|dg¬«      }t
        j                  j                  | |­| |dg¬«      }t
        j                  j                  t        j                  |«       t        j                  |«      ­d|dg¬«      }|
|z   |z   t        j                  |«      z  }
|
j                  |«      }
|dz  }t        j                  |dz  |¬	«      j                  ddd«      }|
|	||dz
  z  z  z  }t        j                  t        j                  |«      t        j                  |«      gd¬
«      j                  «       S )Nr   r   ç      ð?r   r   T)r   Úkeepdim)Úsizer	   r   )r   r   r   r(   ÚnanmeanÚnpÚrandomÚuniformr%   r#   r)   r   r   r   Úfloat)r   r   r2   r3   r4   r5   r6   r7   r
   r   r   ÚdeltaÚdelta_localÚlog_lambdasr   r   r   s                    r   Úcreate_sin_embedding_caperD   I   s®  € ð �‰7�aŠ<Ðˆ<Ø
”—‘˜VÓ$×)Ñ)¨"¨a°Ó3Ñ
3€CØ
�*‰*�Q˜
 AÓ
&€CÙØŒu�}‰}˜S a°Ô6Ñ6ˆáÜ—	‘	×!Ñ!ØÐÐ 0Ð0¸¸:ÀqÐ7Ið "ó 
ˆô —i‘i×'Ñ'ØÐ˜Ð.°f¸jÈ!Ð5Lð (ó 
ˆô —i‘i×'Ñ'Ü�V‰V�IÓÐ¤§¡¨	Ó!2Ð 2¸!¸ZÈÐ9Kð (ó 
ˆð �U‰{˜[Ñ(¬B¯F©F°;Ó,?Ñ?ˆà
�&‰&�‹.€Cà�a‰x€HÜ�<‰<˜˜q™¨Ô0×5Ñ5°a¸¸BÓ?€DØ�: $¨(°Q©,Ñ"7Ñ8Ñ9€EÜ�9‰9ä�I‰I�eÓÜ�I‰I�eÓð	
ð ô÷ �eƒgðr   c                 óD   — t        j                  | «      }||d d …d f   kD  S ©N)r   r   )r   r   s     r   Úget_causal_maskrG   v   s#   € Ü
�,‰,�vÓ
€CØ�’Q˜�W‘ÑÐr   c                 ó¢  — |dv sJ ‚|dk(  rMt        j                  || t         j                  ¬«      }d|dd…d|…f<   t        ||z  | z  «      }	d|d|	…dd…f<   |dk(  rÍt        j                  || t         j                  ¬«      }t        j                  |«      dd…df   }
| |z  |
z  t        j                  | |dz   «      z   j                  «       j                  d| dz
  «      }|j                  d|t        j                  dt         j                  ¬«      j                  |«      «       �n“|d	k(  �r3t        j                  |d
z   | d
z   t         j                  ¬«      }t        j                  |d
z   «      dd…df   }
t        j                  dt        d
| z  dz  dz   «      «      }||dz   z  d
z  j                  «       }t        j                  |j                  d«      dd  |g«      }| |z  |
z  |z   j                  «       j                  d| dz   «      }|j                  d|t        j                  dt         j                  ¬«      j                  |«      «       |dd…dd…f   }nZ|dk(  rUt        j                  |¬«      }|j                  |«       t        j                  | |z  ||¬«      j                  || «      |kD  }j!                  |«      }|S )zn
    When the input of the Decoder has length T1 and the output T2
    The mask matrix has shape (T2, T1)
    )ÚdiagÚjmaskr>   ÚglobalrK   )ÚdtypeTNrI   r   r   rJ   r   ç      à?r   r>   r	   )Ú	generatorr
   )r   r!   Úboolr"   r   ÚlongÚclampÚscatter_ÚonesÚ	expand_asr   ÚflipÚ	GeneratorÚmanual_seedÚrandÚreshaper)   )ÚT1ÚT2Ú	mask_typeÚsparse_attn_windowÚglobal_windowÚmask_random_seedÚsparsityr
   ÚmaskÚline_windowÚrowsÚcolsÚtÚgenes                 r   Úget_elementary_maskrg   {   s€  € ð Ð=Ñ=Ð=Ð=à�HÒÜ�{‰{˜2˜r¬¯©Ô4ˆØ"&ˆŠQ���ÐÑÜ˜-¨"Ñ,¨rÑ1Ó2ˆØ $ˆˆ\ˆkˆ\š1ˆ_Ñà�FÒä�{‰{˜2˜r¬¯©Ô4ˆÜ�|‰|˜BÓ¢ 4 Ñ(ˆà�"‰W�t‰^œeŸl™lÐ,>Ð+>Ð@RÐUVÑ@VÓWÑWß‰T‹Vß‰U�1�b˜1‘fÓð 	ð
 	�‰�a˜œuŸz™z¨!´5·:±:Ô>×HÑHÈÓNÖOà	�gÓ	Ü�{‰{˜2 ™6 2¨¡6´·±Ô<ˆÜ�|‰|˜B ™FÓ#¢A t GÑ,ˆÜ�L‰L˜œC  R¡¨C¡°!Ñ 3Ó4Ó5ˆØ�!�a‘%‰[˜1‰_×!Ñ!Ó#ˆÜ�I‰I˜Ÿ™˜q›	 # 2˜�¨Ð*Ó+ˆØ�R‘˜$‘ Ñ"×(Ñ(Ó*×0Ñ0°°B¸±FÓ;ˆØ�‰�a˜œuŸz™z¨!´5·:±:Ô>×HÑHÈÓNÔOØ�A�b�D˜!˜B˜$�JÑ‰à	�hÒ	Ü�‰ fÔ-ˆØ×ÑÐ)Ô*ä�J‰J�r˜B‘w¨$°vÔ>×FÑFÀrÈ2ÓNØñð 	ð
 �7‰7�6‹?€DØ€Kr   c                 óð   — ddl m} |j                  d«      }	|	D �
cg c]  }
t        | ||
|||||«      ‘Œ }}
t	        j
                  |«      j                  d¬«      dkD  }|j                  |d   «      S c c}
w )z¨
    Return a SparseCSRTensor mask that is a combination of elementary masks
    mask_type can be a combination of multiple masks: for instance "diag_jmask_random"
    r   )ÚSparseCSRTensorÚ_)ÚaxisN)Úxformers.sparseri   Úsplitrg   r   ÚstackÚsumÚ
from_dense)rZ   r[   r\   r]   r^   r_   r`   r
   ri   Ú
mask_typesra   Ú	all_masksÚ
final_masks                r   Úget_maskrt   ²   s‘   € õ 0à—‘ Ó%€Jð öð ô 	ØØØØØØØØõ		
ð€Ið ô —‘˜YÓ'×+Ñ+°Ð+Ó3°aÑ7€Jà×%Ñ% j°Ñ&6Ó7Ð7ùò!s   œA3c            	       óL   ‡ — e Zd Z	 	 ddedededefˆ fd„Zed„ «       Zd„ Zˆ xZ	S )	ÚScaledEmbeddingÚnum_embeddingsÚembedding_dimÚscaleÚboostc                 ó¾   •— t         ‰| �  «        t        j                  ||«      | _        | j                  j
                  xj                  ||z  z  c_        || _        y rF   )ÚsuperÚ__init__ÚnnÚ	EmbeddingÚ	embeddingÚweightÚdatarz   )Úselfrw   rx   ry   rz   Ú	__class__s        €r   r}   zScaledEmbedding.__init__Ø   sH   ø€ ô 	‰ÑÔÜŸ™ n°mÓDˆŒØ�‰×Ñ×"Ò" e¨e¡mÑ3Õ"Øˆ�
r   c                 óH   — | j                   j                  | j                  z  S rF   )r€   r�   rz   )rƒ   s    r   r�   zScaledEmbedding.weightä   s   € à�~‰~×$Ñ$ t§z¡zÑ1Ð1r   c                 ó>   — | j                  |«      | j                  z  S rF   )r€   rz   ©rƒ   Úxs     r   ÚforwardzScaledEmbedding.forwardè   s   € Ø�~‰~˜aÓ  4§:¡:Ñ-Ð-r   )r9   g      @)
Ú__name__Ú
__module__Ú__qualname__r"   r@   r}   Úpropertyr�   r‰   Ú__classcell__©r„   s   @r   rv   rv   ×   sO   ø„ ð
 Øñ
àð
ð ð
ð ð	
ð
 õ
ð ñ2ó ð2ö.r   rv   c                   ó4   ‡ — e Zd ZdZddedefˆ fd„Zd„ Zˆ xZS )Ú
LayerScalez¢Layer scale from [Touvron et al 2021] (https://arxiv.org/pdf/2103.17239.pdf).
    This rescales diagonaly residual outputs close to 0 initially, then learnt.
    ÚchannelsÚinitc                 óÀ   •— t         ‰| �  «        || _        t        j                  t        j                  |d¬«      «      | _        || j                  j                  dd y)z„
        channel_last = False corresponds to (B, C, T) tensors
        channel_last = True corresponds to (T, B, C) tensors
        T)Úrequires_gradN)	r|   r}   Úchannel_lastr~   Ú	Parameterr   r!   ry   r‚   )rƒ   r’   r“   r–   r„   s       €r   r}   zLayerScale.__init__ñ   sE   ø€ ô
 	‰ÑÔØ(ˆÔÜ—\‘\¤%§+¡+¨hÀdÔ"KÓLˆŒ
Ø!ˆ�
‰
�‰™Ñr   c                 ód   — | j                   r| j                  |z  S | j                  d d …d f   |z  S rF   )r–   ry   r‡   s     r   r‰   zLayerScale.forwardû   s1   € Ø×ÒØ—:‘: ‘>Ð!à—:‘:ša ˜gÑ&¨Ñ*Ð*r   )r   F)	rŠ   r‹   rŒ   Ú__doc__r"   r@   r}   r‰   rŽ   r�   s   @r   r‘   r‘   ì   s    ø„ ññ" ð "¨Eõ "ö+r   r‘   c                   ó(   ‡ — e Zd Zˆ fd„Zˆ fd„Zˆ xZS )ÚMyGroupNormc                 ó$   •— t        ‰| �  |i |¤Ž y rF   )r|   r}   )rƒ   ÚargsÚkwargsr„   s      €r   r}   zMyGroupNorm.__init__  s   ø€ Ü‰Ñ˜$Ð) &Ó)r   c                 óf   •— |j                  dd«      }t        ‰| �	  |«      j                  dd«      S )zh
        x: (B, T, C)
        if num_groups=1: Normalisation on all T and C together for each B
        r   r   )r'   r|   r‰   )rƒ   rˆ   r„   s     €r   r‰   zMyGroupNorm.forward  s1   ø€ ð
 �K‰K˜˜1ÓˆÜ‰w‰˜qÓ!×+Ñ+¨A¨qÓ1Ð1r   ©rŠ   r‹   rŒ   r}   r‰   rŽ   r�   s   @r   r›   r›     s   ø„ ô*÷2ð 2r   r›   c                   ób   ‡ — e Zd Zddej                  ddddddddddd	d
ddddfˆ fd„	Zdd„Zˆ xZS )ÚMyTransformerEncoderLayeré   çš™™™™™¹?r   Fçñhãˆµøä>ç-Cëâ6?NrI   é*   éô  é2   çffffffî?c                 óª  •— ||dœ}t         ‰| �  ||||||	||||¬«
       || _        || _        |r|s|| _        || _        || _        || _        |r:t        t        |«      |fd|	i|¤Ž| _
        t        t        |«      |fd|	i|¤Ž| _        d | _        | j                  |z  rt        t        |«      |¬«      | _        |
rt        ||d«      nt        j                   «       | _        |
rt        ||d«      nt        j                   «       | _        |rGt'        |||||r|nd¬«      | _        | j+                  dt-        j.                  d	d	«      «       || _        y y )
N©r
   rL   )
r*   ÚnheadÚdim_feedforwardÚdropoutÚ
activationÚlayer_norm_epsÚbatch_firstÚ
norm_firstr
   rL   Úeps©Ú
num_groupsÚnum_channelsTr   ©r¯   r²   Úauto_sparsityÚsrc_maskr   )r|   r}   Úsparser¹   r\   r]   r^   r`   r›   r"   Únorm1Únorm2Únorm_outr³   r‘   r~   ÚIdentityÚgamma_1Úgamma_2ÚMultiheadAttentionÚ	self_attnÚ__setattr__r   r!   r_   )rƒ   r*   r­   r®   r¯   r°   Ú
group_normr³   r¾   r±   Úlayer_scaleÚinit_valuesr
   rL   r»   r\   r_   r]   r^   r¹   r`   r²   Úfactory_kwargsr„   s                          €r   r}   z"MyTransformerEncoderLayer.__init__  s^  ø€ ð0 %+°UÑ;ˆÜ‰ÑØØØ+ØØ!Ø)Ø#Ø!ØØð 	ô 	
ð ˆŒØ*ˆÔÙÙ Ø!*�”Ø*<�Ô'Ø%2�Ô"Ø$ˆDŒMÙÜ$¤S¨£_°gÑdÀ>ÐdÐUcÑdˆDŒJÜ$¤S¨£_°gÑdÀ>ÐdÐUcÑdˆDŒJàˆŒØ�?‰?˜XÒ%Ü'´3°x³=ÈwÔWˆDŒMá6AŒJ�w ¨TÔ2ÄrÇ{Á{Ã}ð 	Œñ 7BŒJ�w ¨TÔ2ÄrÇ{Á{Ã}ð 	Œñ Ü/Ø˜¨¸[Ù*7™h¸QôˆDŒNð ×Ñ˜Z¬¯©°Q¸Ó):Ô;Ø$4ˆDÕ!ð r   c           
      óP  — |j                   }|}|j                  \  }}}| j                  r„| j                  sx|�J ‚| j                  }|j                  d   |k7  rVt        ||| j                  | j                  | j                  | j                  | j                  |«      }| j                  d|«       | j                  r…|| j                  | j                  | j                  |«      ||«      «      z   }|| j!                  | j#                  | j%                  |«      «      «      z   }| j&                  r| j'                  |«      }|S | j                  || j                  | j                  |||«      «      z   «      }| j%                  || j!                  | j#                  |«      «      z   «      }|S )zw
        if batch_first = False, src shape is (T, B, C)
        the case where batch_first=True is not covered
        r   rº   )r
   Úshaper»   r¹   rº   rt   r\   r]   r^   r_   r`   rÄ   r³   rÀ   Ú	_sa_blockr¼   rÁ   Ú	_ff_blockr½   r¾   )	rƒ   Úsrcrº   Úsrc_key_padding_maskr
   rˆ   ÚTÚBÚCs	            r   r‰   z!MyTransformerEncoderLayer.forwardS  sv  € ð
 —‘ˆØˆØ—'‘'‰ˆˆ1ˆaØ�;Š;˜t×1Ò1ØÐ#Ð#Ð#Ø—}‘}ˆHØ�~‰~˜bÑ! QÒ&Ü#ØØØ—N‘NØ×+Ñ+Ø×&Ñ&Ø×)Ñ)Ø—M‘MØó	�ð × Ñ  ¨XÔ6à�?Š?Ø�D—L‘LØ—‘˜tŸz™z¨!›}¨hÐ8LÓMóñ ˆAð �D—L‘L §¡°·
±
¸1³Ó!>Ó?Ñ?ˆAà�}Š}Ø—M‘M !Ó$�ð ˆð —
‘
Ø�D—L‘L §¡°°8Ð=QÓ!RÓSÑSóˆAð —
‘
˜1˜tŸ|™|¨D¯N©N¸1Ó,=Ó>Ñ>Ó?ˆAàˆr   )NN)rŠ   r‹   rŒ   ÚFÚrelur}   r‰   rŽ   r�   s   @r   r¢   r¢     sN   ø„ ð
 ØØ—6‘6ØØØØØØØØØØØØØØØØõ-A5÷F&r   r¢   c                   ó    ‡ — e Zd Zddej                  ddddddddddd	d
ddddfdedededededededededefˆ fd„Zdd„Z	dd„Z
d„ Zd„ Zˆ xZS )ÚCrossTransformerEncoderLayerr£   r¤   r¥   Fr¦   rI   r§   r¨   r©   rª   Nr*   r­   r®   r¯   r±   rÆ   rÇ   r³   rÅ   r¾   c                 óF  •— ||dœ}t         ‰| �  «        || _        || _        |r|s|| _        || _        || _        || _        |  t        j                  ||||¬«      | _
        t        j                  ||fi |¤Ž| _        t        j                  |«      | _        t        j                  ||fi |¤Ž| _        |	| _        |  |  |  |
rXt#        t%        |
«      |fd|i|¤Ž| _        t#        t%        |
«      |fd|i|¤Ž| _        t#        t%        |
«      |fd|i|¤Ž| _        nWt        j,                  |fd|i|¤Ž| _        t        j,                  |fd|i|¤Ž| _        t        j,                  |fd|i|¤Ž| _        d | _        | j                   |z  rt#        t%        |«      |¬«      | _        |rt1        ||d«      nt        j2                  «       | _        |rt1        ||d«      nt        j2                  «       | _        t        j                  |«      | _        t        j                  |«      | _        t=        |t>        «      r| jA                  |«      | _!        n|| _!        |rJt        |||||r|nd¬«      | _
        |s.| jE                  dtG        jH                  d	d	«      «       || _%        y y y )
Nr¬   )r¯   r²   r´   rµ   Tr   r¸   ra   r   )&r|   r}   r»   r¹   r\   r]   r^   r`   r~   rÂ   Ú
cross_attnÚLinearÚlinear1ÚDropoutr¯   Úlinear2r³   r›   r"   r¼   r½   Únorm3Ú	LayerNormr¾   r‘   r¿   rÀ   rÁ   Údropout1Údropout2Ú
isinstanceÚstrÚ_get_activation_fnr°   rÄ   r   r!   r_   )rƒ   r*   r­   r®   r¯   r°   r±   rÆ   rÇ   r³   rÅ   r¾   r»   r\   r_   r]   r^   r`   r¹   r
   rL   r²   rÈ   r„   s                          €r   r}   z%CrossTransformerEncoderLayer.__init__}  sb  ø€ ð0 %+°UÑ;ˆÜ‰ÑÔàˆŒØ*ˆÔÙÙ Ø!*�”Ø*<�Ô'Ø%2�Ô"Ø$ˆDŒMáÜ×/Ñ/Ø�U G¸ôFˆŒô —y‘y ¨/ÑL¸^ÑLˆŒÜ—z‘z 'Ó*ˆŒÜ—y‘y °'ÑL¸^ÑLˆŒà$ˆŒÙÙÙÙÜ$¤S¨£_°gÑdÀ>ÐdÐUcÑdˆDŒJÜ$¤S¨£_°gÑdÀ>ÐdÐUcÑdˆDŒJÜ$¤S¨£_°gÑdÀ>ÐdÐUcÑdˆD�JäŸ™ gÑT°>ÐTÀ^ÑTˆDŒJÜŸ™ gÑT°>ÐTÀ^ÑTˆDŒJÜŸ™ gÑT°>ÐTÀ^ÑTˆDŒJàˆŒØ�?‰?˜XÒ%Ü'´3°x³=ÈwÔWˆDŒMñ 7BŒJ�w ¨TÔ2ÄrÇ{Á{Ã}ð 	Œñ 7BŒJ�w ¨TÔ2ÄrÇ{Á{Ã}ð 	Œô Ÿ
™
 7Ó+ˆŒÜŸ
™
 7Ó+ˆŒô �j¤#Ô&Ø"×5Ñ5°jÓAˆD�Oà(ˆDŒOáÜ0Ø˜¨¸[Ù*7™h¸Qô@ˆDŒOñ !Ø× Ñ  ¬¯©°Q¸Ó):Ô;Ø(8�Õ%ð !ð	 r   c           
      ó®  — |j                   }|j                  \  }}}|j                  \  }}}| j                  r–| j                  sŠ|�J ‚| j                  }|j                  d   |k7  s|j                  d   |k7  rVt        ||| j                  | j                  | j                  | j                  | j                  |«      }| j                  d|«       | j                  r”|| j                  | j                  | j                  |«      | j!                  |«      |«      «      z   }	|	| j#                  | j%                  | j'                  |	«      «      «      z   }	| j(                  r| j)                  |	«      }	|	S | j                  || j                  | j                  |||«      «      z   «      }	| j!                  |	| j#                  | j%                  |	«      «      z   «      }	|	S )z“
        Args:
            q: tensor of shape (T, B, C)
            k: tensor of shape (S, B, C)
            mask: tensor of shape (T, S)

        r   éþÿÿÿra   )r
   rÊ   r»   r¹   ra   rt   r\   r]   r^   r_   r`   rÄ   r³   rÀ   Ú	_ca_blockr¼   r½   rÁ   rÌ   rÜ   r¾   )
rƒ   ÚqÚkra   r
   rÏ   rÐ   rÑ   ÚSrˆ   s
             r   r‰   z$CrossTransformerEncoderLayer.forwardÒ  s†  € ð —‘ˆØ—'‘'‰ˆˆ1ˆaØ—'‘'‰ˆˆ1ˆaØ�;Š;˜t×1Ò1Ø�<Ð�<Ø—9‘9ˆDØ�z‰z˜"‰~ Ò" d§j¡j°¡n¸Ò&9ÜØØØ—N‘NØ×+Ñ+Ø×&Ñ&Ø×)Ñ)Ø—M‘MØó	�ð × Ñ  ¨Ô.à�?Š?Ø�D—L‘L §¡°·
±
¸1³¸t¿z¹zÈ!»}ÈdÓ!SÓTÑTˆAØ�D—L‘L §¡°·
±
¸1³Ó!>Ó?Ñ?ˆAØ�}Š}Ø—M‘M !Ó$�ð
 ˆð —
‘
˜1˜tŸ|™|¨D¯N©N¸1¸aÀÓ,FÓGÑGÓHˆAØ—
‘
˜1˜tŸ|™|¨D¯N©N¸1Ó,=Ó>Ñ>Ó?ˆAàˆr   c                 óV   — | j                  ||||d¬«      d   }| j                  |«      S )NF)Ú	attn_maskÚneed_weightsr   )r×   rÞ   )rƒ   ræ   rç   rê   rˆ   s        r   rå   z&CrossTransformerEncoderLayer._ca_blockù  s.   € Ø�O‰O˜A˜q !¨yÀuˆOÓMÈaÑPˆØ�}‰}˜QÓÐr   c           	      ó    — | j                  | j                  | j                  | j                  |«      «      «      «      }| j	                  |«      S rF   )rÛ   r¯   r°   rÙ   rß   r‡   s     r   rÌ   z&CrossTransformerEncoderLayer._ff_blockþ  s9   € Ø�L‰L˜Ÿ™ d§o¡o°d·l±lÀ1³oÓ&FÓGÓHˆØ�}‰}˜QÓÐr   c                 óŠ   — |dk(  rt         j                  S |dk(  rt         j                  S t        dj	                  |«      «      ‚)NrÓ   Úgeluz&activation should be relu/gelu, not {})rÒ   rÓ   rî   ÚRuntimeErrorr    )rƒ   r°   s     r   râ   z/CrossTransformerEncoderLayer._get_activation_fn  s;   € Ø˜ÒÜ—6‘6ˆMØ˜6Ò!Ü—6‘6ˆMäÐC×JÑJÈ:ÓVÓWÐWr   rF   )rŠ   r‹   rŒ   rÒ   rÓ   r"   r@   rO   r}   r‰   rå   rÌ   râ   rŽ   r�   s   @r   rÕ   rÕ   |  sË   ø„ ð
  $ØØ—6‘6Ø $Ø!Ø!Ø Ø ØØØØØØØØØØØñ-S9àðS9ð ðS9ð ð	S9ð
 ðS9ð ðS9ð ðS9ð ðS9ð ðS9ð ðS9ð õS9ój$óN ò
 öXr   rÕ   c            ?       ó
  ‡ — e Zd Zddddddddddddd	dd
ddddddg d¢ddddddddfdededededededededededededed ed!ed"ej                  e   d#ed$ed%ed&ed'ed(ed)e	d*ed+ed,ed-ed.ed/ed0ed1ef>ˆ fd2„Z
d3„ Zd4„ Zd5„ Zˆ xZS )6ÚCrossTransformerEncoderr   g      @é   é   Fr   iè  Tç     ˆÃ@Nr   r9   )g     ˆ³@r9   gffffffö?rI   r§   r¨   r©   rª   r   ÚembÚhidden_scaleÚ	num_headsÚ
num_layersÚcross_firstr¯   Úmax_positionsÚnorm_inÚnorm_in_grouprÅ   r³   r¾   r   Úweight_decayÚlrrÆ   rî   Úsin_random_shiftÚweight_pos_embedÚcape_mean_normalizeÚcape_augmentÚcape_glob_loc_scaleÚsparse_self_attnÚsparse_cross_attnr\   r_   r]   r^   r¹   r`   c                  ób  •— t         ‰&| �  «        	 ||z  dk(  sJ ‚t        ||z  «      } || _        |rdnd| _        || _        || _        || _        || _        || _	        |dk(  r|| _
        || _        || _        |dk(  rt        ||d¬«      | _        || _        |rt         j"                  nt         j$                  }!|  |  |	r5t'        j(                  |«      | _        t'        j(                  |«      | _        ni|
r5t/        t        |
«      |«      | _        t/        t        |
«      |«      | _        n2t'        j0                  «       | _        t'        j0                  «       | _        t'        j2                  «       | _        t'        j2                  «       | _        i d|“d|“d	| “d
|“d|!“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“dd“}"t9        |"«      }#|#j;                  d|i«       t9        |"«      }$|$j;                  d|i«       t=        |«      D ]¥  }%|%dz  | j                  k(  rI| j4                  j?                  tA        di |#¤Ž«       | j6                  j?                  tA        di |#¤Ž«       Œ^| j4                  j?                  tC        di |$¤Ž«       | j6                  j?                  tC        di |$¤Ž«       Œ§ y )Nr   r   ÚcapeÚscaledgš™™™™™É?)ry   r*   r­   r®   r¯   r°   rÅ   r³   r¾   rÆ   r\   r_   r]   r^   r`   r¹   r²   Tr»   r   © )"r|   r}   r"   rø   Úclassic_parityrõ   r   rý   r   rÿ   r  r  r  rv   Úposition_embeddingsrþ   rÒ   rî   rÓ   r~   rÝ   rû   Ú	norm_in_tr›   r¿   Ú
ModuleListÚlayersÚlayers_tÚdictÚupdateÚrangeÚappendr¢   rÕ   )'rƒ   r   rõ   rö   r÷   rø   rù   r¯   rú   rû   rü   rÅ   r³   r¾   r   rý   rþ   rÆ   rî   rÿ   r   r  r  r  r  r  r\   r_   r]   r^   r¹   r`   Ú
hidden_dimr°   Úkwargs_commonÚkwargs_classic_encoderÚkwargs_cross_encoderÚidxr„   s'                                         €r   r}   z CrossTransformerEncoder.__init__  sù  ø€ ôD 	‰ÑÔð	à�Y‰ !Ò#Ð#Ð#ä˜˜|Ñ+Ó,ˆ
à$ˆŒñ $/™a°AˆÔØˆŒØ$ˆŒØ(ˆÔØ 0ˆÔØ 0ˆÔØ�&Š=Ø':ˆDÔ$Ø ,ˆDÔØ':ˆDÔ$Ø�(Š?Ü'6°}ÀcÐQTÔ'UˆDÔ$àˆŒá'+œQŸVšV´·±ˆ
áÙÙÜŸ<™<¨Ó,ˆDŒLÜŸ\™\¨#Ó.ˆD�NÙÜ&¤s¨=Ó'9¸3Ó?ˆDŒLÜ(¬¨]Ó);¸SÓAˆD�NäŸ;™;›=ˆDŒLÜŸ[™[›]ˆDŒNô —m‘m“oˆŒäŸ™›ˆŒð
Ø�sð
à�Yð
ð ˜zð
ð �wð	
ð
 ˜*ð
ð ˜*ð
ð ˜*ð
ð ˜ð
ð ˜;ð
ð ˜ð
ð Ð 0ð
ð !Ð"4ð
ð ˜]ð
ð ˜ð
ð ˜]ð
ð  ˜4ð!
ˆô& "& mÓ!4ÐØ×%Ñ%ØÐ&ð'
ô 	ô  $ MÓ2ÐØ×#Ñ#ØÐ'ð%
ô 	ô ˜Ó$ò 	ˆCØ�Q‰w˜$×-Ñ-Ò-à—‘×"Ñ"Ô#<Ñ#VÐ?UÑ#VÔWØ—‘×$Ñ$Ü-ÑGÐ0FÑGõð
 —‘×"Ñ"Ô#?Ñ#WÐBVÑ#WÔXà—‘×$Ñ$Ü0ÑHÐ3GÑHõñ	r   c                 óø  — |j                   \  }}}}t        ||||j                  | j                  «      }t	        |d«      }t	        |d«      }| j                  |«      }|| j                  |z  z   }|j                   \  }}}t	        |d«      }| j                  ||||j                  «      }	t	        |	d«      }	| j                  |«      }|| j                  |	z  z   }t        | j                  «      D ]m  }
|
dz  | j                  k(  r+ | j                  |
   |«      } | j                  |
   |«      }Œ@|} | j                  |
   ||«      } | j                  |
   ||«      }Œo t	        |d|¬«      }t	        |d«      }||fS )Nzb c fr t1 -> b (t1 fr) czb c t2 -> b t2 czt2 b c -> b t2 cr   zb (t1 fr) c -> b c fr t1)Út1zb t2 c -> b c t2)rÊ   r1   r
   r   r   rû   r   Ú_get_pos_embeddingr  r  rø   r
  r  r  )rƒ   rˆ   ÚxtrÐ   rÑ   ÚFrrZ   Ú
pos_emb_2dr[   Úpos_embr  Úold_xs               r   r‰   zCrossTransformerEncoder.forwardˆ  s‚  € Ø—w‘w‰ˆˆ1ˆb�"Ü,Øˆr�2�q—x‘x §¡ó
ˆ
ô ˜zÐ+EÓFˆ
Ü�aÐ3Ó4ˆØ�L‰L˜‹OˆØ�×%Ñ%¨
Ñ2Ñ2ˆà—8‘8‰ˆˆ1ˆbÜ�rÐ-Ó.ˆØ×)Ñ)¨"¨a°°A·H±HÓ=ˆÜ˜GÐ%7Ó8ˆØ�^‰^˜BÓˆØ�$×'Ñ'¨'Ñ1Ñ1ˆä˜Ÿ™Ó)ò 	3ˆCØ�Q‰w˜$×-Ñ-Ò-Ø$�D—K‘K Ñ$ QÓ'�Ø'�T—]‘] 3Ñ'¨Ó+‘à�Ø$�D—K‘K Ñ$ Q¨Ó+�Ø'�T—]‘] 3Ñ'¨¨EÓ2‘ð	3ô �aÐ3¸Ô;ˆÜ�rÐ-Ó.ˆØ�"ˆuˆr   c                 óZ  — | j                   dk(  r>t        j                  | j                  dz   «      }t	        ||||| j
                  ¬«      }|S | j                   dk(  r�| j                  r\t        ||||| j
                  | j                  | j                  | j                  d   | j                  d   | j                  d   ¬«
      }|S t        ||||| j
                  | j                  d¬	«      }|S | j                   d
k(  r/t        j                  ||¬«      }| j                  |«      d d …d f   }S )Nr   r   )r   r
   r   r  r   r   )r
   r   r3   r4   r5   r6   r7   F)r
   r   r3   r4   r  r	   )rõ   r>   Ú	randrangerÿ   r   r   ÚtrainingrD   r  r  r  r   r   r  )rƒ   rÏ   rÐ   rÑ   r
   r   r  r   s           r   r  z*CrossTransformerEncoder._get_pos_embedding¦  s-  € Ø�8‰8�uÒÜ×$Ñ$ T×%:Ñ%:¸QÑ%>Ó?ˆEÜ*Ø�1˜E¨&¸T¿_¹_ôˆGð@ ˆð; �X‰X˜ÒØ�}Š}Ü3ØØØØ!Ø#Ÿ™Ø#'×#;Ñ#;Ø ×-Ñ-Ø%)×%=Ñ%=¸aÑ%@Ø$(×$<Ñ$<¸QÑ$?Ø"×6Ñ6°qÑ9ô�ð6 ˆô 4ØØØØ!Ø#Ÿ™Ø#'×#;Ñ#;Ø!ô�ð ˆð	 �X‰X˜Ò!Ü—,‘,˜q¨Ô0ˆCØ×.Ñ.¨sÓ3²A°t°GÑ<ˆGàˆr   c                 óˆ   — t        | j                  «       «      | j                  dœ}| j                  �| j                  |d<   |S )N)Úparamsrý   rþ   )ÚlistÚ
parametersrý   rþ   )rƒ   Úgroups     r   Úmake_optim_groupz(CrossTransformerEncoder.make_optim_groupË  s:   € Ü §¡Ó 1Ó2ÀD×DUÑDUÑVˆØ�7‰7ÐØŸ'™'ˆE�$‰KØˆr   )rŠ   r‹   rŒ   r"   rá   r@   rO   ÚtpÚOptionalr&  r}   r‰   r  r)  rŽ   r�   s   @r   rñ   rñ     sÕ  ø„ ð Ø!ØØØ!ØØ!ØØ#ØØ ØØ#Ø!Ø!%Ø!ØØ !Ø"%Ø$(Ø!Ú$6Ø!&Ø"'ØØ "Ø"%ØØ#ØñAwàðwð ðwð ð	wð
 ðwð ðwð ðwð ðwð ðwð ðwð ðwð ðwð ðwð ðwð ðwð  ð!wð" �K‰K˜Ñð#wð$ ð%wð& ð'wð( ð)wð*  ð+wð, "ð-wð. ð/wð0 "ð1wð2 ð3wð4  ð5wð6 ð7wð8 ð9wð:  ð;wð< ð=wð> ð?wð@ õAwòrò<#öJr   rñ   c                   ó@   ‡ — e Zd Z	 	 	 	 	 	 	 	 dˆ fd„	Z	 	 	 	 dd„Zˆ xZS )rÂ   c                 ó$  •— t         ‰| �  «        |
€J d«       ‚|| _        t        j                  j                  |||¬«      | _        t        j                  j                  |||¬«      | _        t        j                  j                  |||¬«      | _        t        j                  j                  |«      | _
        t        j                  j                  |||«      | _        t        j                  j                  |«      | _        |	| _        |
| _        y )Nzsanity check)Úbias)r|   r}   r÷   r   r~   rØ   ræ   rç   ÚvrÚ   Ú	attn_dropÚprojÚ	proj_dropr²   r¹   )rƒ   Ú	embed_dimr÷   r¯   r.  Úadd_bias_kvÚadd_zero_attnÚkdimÚvdimr²   r¹   r„   s              €r   r}   zMultiheadAttention.__init__Ö  sÊ   ø€ ô 	‰ÑÔØÐ(Ð8¨.Ó8Ð(Ø"ˆŒÜ—‘—‘ ¨I¸D�ÓAˆŒÜ—‘—‘ ¨I¸D�ÓAˆŒÜ—‘—‘ ¨I¸D�ÓAˆŒÜŸ™×)Ñ)¨'Ó2ˆŒÜ—H‘H—O‘O I¨y¸$Ó?ˆŒ	ÜŸ™×)Ñ)¨'Ó2ˆŒØ&ˆÔØ*ˆÕr   c                 ó®  — | j                   s9|j                  ddd«      }|j                  ddd«      }|j                  ddd«      }|j                  \  }}	}
|j                  \  }}}
| j                  |«      j	                  ||	| j
                  |
| j
                  z  «      j                  dddd«      }|j                  dd«      }| j                  |«      j	                  ||| j
                  |
| j
                  z  «      j                  dddd«      }|j                  dd«      }| j                  |«      j	                  ||| j
                  |
| j
                  z  «      j                  dddd«      }|j                  dd«      }| j                  r|�J ‚t        |||| j                  ¬«      }nt        ||||| j                  ¬«      }|j	                  || j
                  |	|
| j
                  z  «      }|j                  dd«      j	                  ||	|
«      }| j                  |«      }| j                  |«      }| j                   s|j                  ddd«      }|d fS )Nr   r   r   é   )r`   )r¯   )r²   ÚpermuterÊ   ræ   rY   r÷   Úflattenrç   r/  r¹   Údynamic_sparse_attentionÚscaled_dot_product_attentionr0  r'   r1  r2  )rƒ   ÚqueryÚkeyÚvalueÚkey_padding_maskrë   rê   Úaverage_attn_weightsrÐ   ÚN_qrÑ   ÚN_kræ   rç   r/  rˆ   s                   r   r‰   zMultiheadAttention.forwardï  s  € ð ×ÒØ—M‘M ! Q¨Ó*ˆEØ—+‘+˜a  AÓ&ˆCØ—M‘M ! Q¨Ó*ˆEØ—K‘K‰	ˆˆ3�Ø—I‘I‰	ˆˆ3�ð �F‰F�5‹Mß‰W�Q˜˜TŸ^™^¨Q°$·.±.Ñ-@ÓAß‰W�Q˜˜1˜aÓ ð 	
ð
 �I‰I�a˜‹Oˆà�F‰F�3‹Kß‰W�Q˜˜TŸ^™^¨Q°$·.±.Ñ-@ÓAß‰W�Q˜˜1˜aÓ ð 	
ð
 �I‰I�a˜‹Oˆà�F‰F�5‹Mß‰W�Q˜˜TŸ^™^¨Q°$·.±.Ñ-@ÓAß‰W�Q˜˜1˜aÓ ð 	
ð
 �I‰I�a˜‹Oˆà×ÒØÐ$Ð$Ð$Ü(¨¨A¨q¸4×;MÑ;MÔN‰Aä,¨Q°°1°iÈÏÉÔXˆAØ�I‰I�a˜Ÿ™¨¨a°4·>±>Ñ.AÓBˆà�K‰K˜˜1Ó×%Ñ% a¨¨aÓ0ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØ×ÒØ—	‘	˜!˜Q Ó"ˆAØ�$ˆwˆr   )r   TFFNNFN)NTNTr    r�   s   @r   rÂ   rÂ   Õ  s5   ø„ ð
 ØØØØØØØõ+ð< ØØØ!÷1r   rÂ   c                 óÈ   — ddl m} | |j                  d«      dz  z  }  || |j                  dd«      |«      }t        j
                  j                  j                  |d«      }|S )Nr   )Úmasked_matmulr   rM   rä   )Úxformers.opsrF  r;   r'   r   r~   Ú
functionalÚsoftmax)ræ   rç   Úatt_maskrF  Úatts        r   Úscaled_query_key_softmaxrL  #  sV   € Ý*Ø	ˆQ�V‰V�B‹Z˜CÑÑ€AÙ
˜˜1Ÿ;™; r¨2Ó.°Ó
9€CÜ
�(‰(×
Ñ
×
%Ñ
% c¨2Ó
.€CØ€Jr   c                 ó<   — t        | ||¬«      } ||«      }||z  }|S )N)rJ  )rL  )ræ   rç   r/  rJ  r¯   rK  Úys          r   r=  r=  +  s'   € Ü
" 1 a°(Ô
;€CÙ
�#‹,€CØˆa‰€AØ€Hr   c                 óæ   — t        j                  d| |«      }t        j                  || gd¬«      }|j                  d¬«      }|j	                  ddd«      j                  «       j                  «       S )Nzbtf,bfhi->bhtir   r   r   r   r   )r   Úeinsumr   Úargmaxr:  ÚbyteÚ
contiguous)rˆ   ÚRÚqqÚbucketss       r   Ú_compute_bucketsrW  2  s`   € Ü	�‰Ð&¨¨1Ó	-€BÜ	�‰�B˜˜�9 "Ô	%€BØ�i‰i˜BˆiÓ€Gà�?‰?˜1˜a Ó#×(Ñ(Ó*×5Ñ5Ó7Ð7r   c                 óŽ  — ddl m}m} d}d}	| ||fD �
cg c]  }
|
j                  «       ‘Œ c}
\  } }}t	        j
                  «       5  t	        j                  d| j                  d   ||	dz  | j                  ¬«      }t        | |«      }t        ||«      } |||||«      \  }}d d d «        || |||«      S c c}
w # 1 sw Y   ŒxY w)	Nr   )Úfind_locationsÚ!sparse_memory_efficient_attentioné    r   r   r   r   r	   )
rG  rY  rZ  rS  r   Úno_gradÚrandnrÊ   r
   rW  )r>  r?  r@  r`   Úinfer_sparsityÚ	attn_biasrY  rZ  Ún_hashesÚ	proj_sizerˆ   rT  Úbucket_queryÚ
bucket_keyÚrow_offsetsÚcolumn_indicess                   r   r<  r<  :  sÐ   € çNØ€HØ€IØ27¸¸eÐ1DÖE¨A˜Ÿ™�ÒEÑ€Eˆ3�Ü	�‰‹ñ @Ü�K‰K˜˜5Ÿ;™; r™?¨H°iÀ1±nÈUÏ\É\ÔZˆÜ'¨¨qÓ1ˆÜ% c¨1Ó-ˆ
Ù&4Ø˜* h°ó'@Ñ#ˆ�^÷	@ñ -Øˆs�E˜;¨¸	óCð Cùò F÷@ð @ús   ”B6ÁAB;Â;C)r   Úcpué'  )rf  rg  )r   r   r9   rf  rô   )TN)&r>   Útypingr*  r   Útorch.nnr~   Útorch.nn.functionalrH  rÒ   Únumpyr=   r$   Úeinopsr   r"   r   r1   rO   r@   rá   rD   rG   rg   rt   ÚModulerv   r‘   Ú	GroupNormr›   ÚTransformerEncoderLayerr¢   rÕ   rñ   rÂ   rL  r=  rW  r<  r	  r   r   ú<module>rp     si  ðó Û ã Ý ß Ð Û Û Ý ð EJñØðØðØ"%óó$!"ðT "Ø ØØØñ*Øð*à	ð*ð ð*ð ð	*ð
 ð*ð ð*ð ð*ð ð*ð ð*ð ó*òZò
4òn"8ôJ.�b—i‘iô .ô*+�—‘ô +ô,
2�"—,‘,ô 
2ôj × :Ñ :ô jôZLX 2§9¡9ô LXôdA˜bŸi™iô AôNK˜Ÿ™ô Kò\òò8ôCr   