ó
    UShåm  ã                   ó   • S SK r S SKrS SKJs  Jr  S SKJrJr  S SKJrJ	r	  S SK
JrJr  SSKJrJrJr  SSKJr  S SKJr  \R*                  " 5       (       a  S SKrS SKrS S	KJr  S SKr\R8                  R:                  R<                  r\R>                  (       a  \ " S
5        Sr!OSr!S r"S r#S r$S r%S r& " S S\RN                  5      r( " S S\RN                  5      r)S-S jr*S.S jr+S.S jr,S.S jr-Sr. \R^                  r0\0Rc                  S5      =(       d)    \0Rc                  S5      =(       d    \0Rc                  S5      r.S.S jr2S.S jr3\+r4\R*                  " 5       (       a  \ " S 5        \2r4OG\Rj                  " 5       (       a  \ " S!5        \3r4O&\RZ                  (       a  \ " S"5        \-r4O
\ " S#5        \,r4\4r6S/S$ jr7 " S% S&\RN                  5      r8 " S' S(\RN                  5      r9 " S) S*\RN                  5      r: " S+ S,\:5      r;g!    NÊ= f)0é    N)ÚnnÚeinsum)Ú	rearrangeÚrepeat)ÚOptionalÚAnyé   )Ú
checkpointÚAlphaBlenderÚtimestep_embedding)Úefficient_dot_product_attention)Úmodel_management)Úargsz disabling upcasting of attentionÚfp16Úfp32c                 ó
   • U S L$ ©N© )Úvals    ÚE/home/gothic/public_html/Fooocus/ldm_patched/ldm/modules/attention.pyÚexistsr      s   € Ø�dˆ?Ðó    c                 óN   • U  Vs0 s H  oS_M     snR                  5       $ s  snf )NT)Úkeys)ÚarrÚels     r   Úuniqr   !   s%   € Ù!Ó
"šc˜ˆtŠ8™cÑ
"×
'Ñ
'Ó
)Ð)ùÒ
"s   …"c                 ó*   • [        U 5      (       a  U $ U$ r   )r   )r   Úds     r   Údefaultr    %   s   € Üˆc‡{�{Øˆ
Ø€Hr   c                 óX   • [         R                  " U R                  5      R                  * $ r   )ÚtorchÚfinfoÚdtypeÚmax©Úts    r   Úmax_neg_valuer(   +   s   € Ü�KŠK˜Ÿ™Ó ×$Ñ$Ð$Ð$r   c                 ó|   • U R                   S   nS[        R                  " U5      -  nU R                  U* U5        U $ )Néÿÿÿÿr	   )ÚshapeÚmathÚsqrtÚuniform_)ÚtensorÚdimÚstds      r   Úinit_r2   /   s7   € Ø
�,‰,�rÑ
€CØ
Œd�iŠi˜‹nÑ
€CØ
‡O�O�S�D˜#ÔØ€Mr   c                   ó8   ^ • \ rS rSrSS\4U 4S jjrS rSrU =r$ )ÚGEGLUé7   Nc                 óR   >• [         TU ]  5         UR                  XS-  X4S9U l        g )Né   ©r$   Údevice)ÚsuperÚ__init__ÚLinearÚproj)ÚselfÚdim_inÚdim_outr$   r9   Ú
operationsÚ	__class__s         €r   r;   ÚGEGLU.__init__8   s)   ø€ Ü‰ÑÔØ×%Ñ% f¸©kÀÐ%ÐVˆ�	r   c                 óv   • U R                  U5      R                  SSS9u  pU[        R                  " U5      -  $ )Nr7   r*   ©r0   )r=   ÚchunkÚFÚgelu)r>   ÚxÚgates      r   ÚforwardÚGEGLU.forward<   s4   € Ø—)‘)˜A“,×$Ñ$ Q¨BÐ$Ð/‰ˆØ”1—6’6˜$“<ÑÐr   )r=   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úopsr;   rK   Ú__static_attributes__Ú__classcell__©rB   s   @r   r4   r4   7   s   ø† Ø.2¸4ÈC÷ W÷ ð  r   r4   c                   ó@   ^ • \ rS rSrSSSSSS\4U 4S jjrS rSrU =r$ )	ÚFeedForwardéA   Né   Fç        c	                 ód  >• [         TU ]  5         [        X-  5      n	[        X!5      nU(       d8  [        R
                  " UR                  XXgS9[        R                  " 5       5      O
[        XXgUS9n
[        R
                  " U
[        R                  " U5      UR                  X’XgS95      U l
        g )Nr8   )r$   r9   rA   )r:   r;   Úintr    r   Ú
Sequentialr<   ÚGELUr4   ÚDropoutÚnet)r>   r0   r@   ÚmultÚgluÚdropoutr$   r9   rA   Ú	inner_dimÚ
project_inrB   s              €r   r;   ÚFeedForward.__init__B   sš   ø€ Ü‰ÑÔÜ˜™
“Oˆ	Ü˜'Ó'ˆö ô —]’]Ø×Ñ˜c°EÐÐIÜ�GŠG‹Iô
ô   °eÐWaÑbð 	ô
 —=’=ØÜ�JŠJ�wÓØ×Ñ˜i¸ÐÐMó
ˆ�r   c                 ó$   • U R                  U5      $ r   ©r`   )r>   rI   s     r   rK   ÚFeedForward.forwardQ   s   € Ø�x‰x˜‹{Ðr   rh   rM   rU   s   @r   rW   rW   A   s#   ø† Ø$(¨q°eÀRÈtÐ\`Ðmp÷ 
÷ð r   rW   c           	      óD   • [         R                  R                  SU SSXS9$ )Né    ç�íµ ÷Æ°>T©Ú
num_groupsÚnum_channelsÚepsÚaffiner$   r9   )r"   r   Ú	GroupNorm)Úin_channelsr$   r9   s      r   Ú	Normalizert   T   s$   € Ü�8‰8×Ñ¨¸+È4ÐX\ÐdiÐÐyÐyr   c                 ó  ^^^• U R                   u  mnmTT-  mTS-  nTn[        UUU4S jXU45      u  pn[        S:X  a-  [        SU R	                  5       UR	                  5       5      U-  nO[        SX5      U-  nA A[        U5      (       ax  UR                  [        R                  :X  aV  [        US5      n[        R                  " UR                  5      R                  * n	[        USUS9nUR                  U) U	5        OX„-  nUR                  SS	9n[        S
UR                  UR                  5      U5      n
U
R!                  S5      R#                  TTST5      R%                  SSSS5      R#                  TSTT-  5      n
U
$ )Nç      à¿c                 ó²   >• U R                  S5      R                  TSTT5      R                  SSSS5      R                  TT-  ST5      R                  5       $ ©Né   r*   r   r7   r	   ©Ú	unsqueezeÚreshapeÚpermuteÚ
contiguous©r'   ÚbÚdim_headÚheadss    €€€r   Ú<lambda>Ú!attention_basic.<locals>.<lambda>^   óK   ø€ �!—+‘+˜a“.ß	‰��B˜˜xÓ	(ß	‰��A�q˜!Ó	ß	‰��U‘˜B Ó	)ß	‰‹ð	r   r   úb i d, b j d -> b i jzb ... -> b (...)zb j -> (b h) () j)Úhr*   rE   úb i j, b j d -> b i dr   r7   r	   ry   )r+   ÚmapÚ_ATTN_PRECISIONr   Úfloatr   r$   r"   Úboolr   r#   r%   r   Úmasked_fill_ÚsoftmaxÚtor{   r|   r}   )ÚqÚkÚvr‚   ÚmaskÚ_Úscaler‡   Úsimr(   Úoutr€   r�   s      `       @@r   Úattention_basicr˜   W   si  ú€ Ø—W‘W�N€A€qˆ(Ø�Ñ€HØ˜Ñ€Eà€AÜö	ð
 
ˆqˆ	ó�G€Aˆ!ô ˜ÓÜÐ,¨a¯g©g«i¸¿¹»ÓCÀeÑK‰äÐ,¨aÓ3°eÑ;ˆà	ˆ1äˆd‡|�|Ø�:‰:œŸ™Ó#Ü˜TÐ#5Ó6ˆDÜ"Ÿ[š[¨¯©Ó3×7Ñ7Ð7ˆMÜ˜$Ð 3°qÑ9ˆDØ×Ñ˜d˜U MÕ2à‰KˆCð �+‰+˜"ˆ+Ð
€Cä
Ð(¨#¯&©&°·±«/¸1Ó
=€Cà�‰�aÓß	‰��E˜2˜xÓ	(ß	‰��A�q˜!Ó	ß	‰��B˜ Ñ(Ó	)ð	 ð €Jr   c                 ó¶  • U R                   u  pVnXs-  nUS-  nU R                  S5      R                  USX75      R                  SSSS5      R                  XS-  SU5      n UR                  S5      R                  USX75      R                  SSSS5      R                  XS-  SU5      nUR                  S5      R                  USX75      R                  SSSS5      R                  XS-  US5      nU R                  n	[
        S:H  =(       a    U R                  [        R                  :g  n
U
(       a2  [        R                  " [        R                  5      R                  S-  nO-[        R                  " U R                  5      R                  S-  nU R                   u  pÍnUR                   u    pnXË-  U-  U-  n[        R                  " U R                  S	5      u  nnS nS nS nS
 H  nUXË-  U-  S-  -  nUU:¼  d  M  UnUn  O   Uc  Sn[        U UUUUUSU
US9	nUR                  U	5      nUR                  SSU45      R!                  SS5      R#                  SS9nU$ )Nrv   ry   r*   r   r7   r	   r   é   T)i   i   i   é   é   g      @r›   F)Úquery_chunk_sizeÚkv_chunk_sizeÚkv_chunk_size_minÚuse_checkpointÚupcast_attentionr“   )Ú	start_dim)r+   r{   r|   r}   r$   rŠ   r"   Úfloat32r#   Úbitsr   Úget_free_memoryr9   r   r�   Ú	unflattenÚ	transposeÚflatten)ÚqueryÚkeyÚvaluer‚   r“   r€   r”   r�   r•   r$   r¡   Úbytes_per_tokenÚbatch_x_headsÚq_tokensÚk_tokensÚqk_matmul_size_bytesÚmem_free_totalÚmem_free_torchrŸ   rž   r�   rI   ÚcountÚhidden_statess                           r   Úattention_sub_quadrµ   „   s^  € Ø—[‘[�N€Aˆ(ØÑ€Hà˜Ñ€EØ�O‰O˜AÓ×&Ñ& q¨"¨eÓ>×FÑFÀqÈ!ÈQÐPQÓR×ZÑZÐ[\Ñ[dÐfhÐjrÓs€EØ�O‰O˜AÓ×&Ñ& q¨"¨eÓ>×FÑFÀqÈ!ÈQÐPQÓR×ZÑZÐ[\Ñ[dÐfhÐjrÓs€Eà
�-‰-˜Ó
×
"Ñ
" 1 b¨%Ó
:×
BÑ
BÀ1ÀaÈÈAÓ
N×
VÑ
VÐWXÑW`ÐbjÐlnÓ
o€Cà�K‰K€EÜ&¨Ñ/×P°E·K±KÄ5Ç=Á=Ñ4PÐÞÜŸ+š+¤e§m¡mÓ4×9Ñ9¸1Ñ<‰äŸ+š+ e§k¡kÓ2×7Ñ7¸Ñ:ˆØ!&§¡Ñ€M˜QØ—Y‘Y�N€A€qØ(Ñ:¸XÑEÈÑPÐä%5×%EÒ%EÀeÇlÁlÐTXÓ%YÑ"€N�NàÐØ€MØÐã)ˆØ -Ñ"AÀAÑ"EÈÑ"KÑLˆØ�HÕØ$ˆMØ ÐÙñ *ð ÑØÐä3ØØØØ)Ø#Ø+ØØ)Øñ
€Mð "×$Ñ$ UÓ+€Mà!×+Ñ+¨A°°E¨{Ó;×EÑEÀaÈÓJ×RÑRÐ]^ÐRÐ_€MØÐr   c                 ó–  ^^^• U R                   u  mnmTT-  mTS-  nTn[        UUU4S jXU45      u  pn[        R                  " U R                   S   U R                   S   UR                   S   U R                  U R
                  S9n[        R                  " U R                  5      n	[        S:X  a  Sn
OU R                  5       n
S	nU R                   S   U R                   S   -  UR                   S   -  U
-  nS
nXÍ-  nSnXé:”  a0  S[        R                  " [        R                  " Xé-  S5      5      -  nUS:”  ai  [        R                  " [        R                  " [        R                  " U	S-  5      5      S-  5      S-  n[        SU SU SUS-  U-  S SX›-  S S3	5      eSnSn  U R                   S   U-  S:X  a  U R                   S   U-  OU R                   S   n[!        SU R                   S   U5       Hô  nUU-   n[        S:X  aT  [        R"                  " SSS9   [%        SU S S 2UU24   R'                  5       UR'                  5       5      U-  nS S S 5        O[%        SU S S 2UU24   U5      U-  nUb0  [)        UR                   5      S:X  a	  WUUU -  nOWUS S 2UU24   -  nWR+                  SS9R-                  UR
                  5      nASn[%        SUU5      US S 2UU24'   AMö      A AAUR5                  S5      R7                  TTST5      R9                  SSSS
5      R7                  TSTT-  5      nU$ ! , (       d  f       NÖ= f! [        R.                   a[  nUS:X  aN  [        R0                  " S5        US:X  a  Sn[3        S5         S nAGMõ  US-  nUS:”  a  Ue[3        SU5         S nAO
UeS nAff = fGM  )Nrv   c                 ó²   >• U R                  S5      R                  TSTT5      R                  SSSS5      R                  TT-  ST5      R                  5       $ rx   rz   r   s    €€€r   rƒ   Ú!attention_split.<locals>.<lambda>À   r…   r   r   r	   r7   )r9   r$   r   rY   i   @ry   é@   g      @rš   z5Not enough memory, use lower resolution (max approx. rI   z	). Need: z0.1fzGB free, Have:zGB freeFTÚcuda)ÚenabledÚdevice_typer†   r*   rE   rˆ   z4out of memory error, emptying cache and trying againz6out of memory error, increasing steps and trying again)r+   r‰   r"   Úzerosr9   r$   r   r¥   rŠ   Úelement_sizer,   ÚceilÚlogÚfloorr-   ÚRuntimeErrorÚrangeÚautocastr   r‹   ÚlenrŽ   r�   ÚOOM_EXCEPTIONÚsoft_empty_cacheÚprintr{   r|   r}   )r�   r‘   r’   r‚   r“   r”   r•   r‡   Úr1r±   r¾   ÚgbÚtensor_sizeÚmodifierÚmem_requiredÚstepsÚmax_resÚfirst_op_doneÚcleared_cacheÚ
slice_sizeÚiÚendÚs1Ús2Úer€   r�   s      `                     @@r   Úattention_splitrØ   ¹   sÙ  ú€ Ø—W‘W�N€A€qˆ(Ø�Ñ€HØ˜Ñ€Eà€AÜö	ð
 
ˆqˆ	ó�G€Aˆ!ô 
�Š�Q—W‘W˜Q‘Z §¡¨¡¨Q¯W©W°Q©ZÀÇÁÐPQ×PWÑPWÑ	X€Bä%×5Ò5°a·h±hÓ?€Nä˜ÓØ‰à—~‘~Ó'ˆà	€BØ—'‘'˜!‘*˜qŸw™w q™zÑ)¨A¯G©G°A©JÑ6¸ÑE€KØ€HØÑ)€LØ€Eð Ó$Ø”D—I’IœdŸhšh |Ñ'DÀaÓHÓIÑJˆð ˆrƒzÜ—*’*œTŸYšY¤t§y¢y°À#Ñ1EÓ'FÓGÈ!ÑKÓLÈrÑQˆÜÐRÐSZÐR[Ð[\Ð]dÐ\eð f%Ø%1°"¡_°RÑ%7¸Ð$=¸^ÈNÑL]Ð^bÐKcÐcjðló mð 	mð €MØ€MØ
ð#	Ø12·±¸±¸eÑ1CÈÓ0I˜Ÿ™ ™ uÒ,ÈqÏwÉwÐWXÉzˆJÜ˜1˜aŸg™g a™j¨*Ö5�Ø˜*‘n�Ü" VÓ+ÜŸš°ÀVÓLÜ#Ð$;¸QºqÀ!ÀCÀ%¸x¹[×=NÑ=NÓ=PÐRS×RYÑRYÓR[Ó\Ð_dÑd˜÷ MÐLô  Ð 7¸º1¸aÀ¸e¸8¹ÀaÓHÈ5ÑP�BàÑ#Ü˜4Ÿ:™:“¨!Ó+Ø˜d 1 S˜kÑ)™à˜d¢1 a¨ e 8™nÑ,˜à—Z‘Z B�ZÐ'×*Ñ*¨1¯7©7Ó3�ØØ $�ä%Ð&=¸rÀ1ÓE�’1�a˜�e�8‘Úñ' 6ð( ð 	
ˆ1ˆað 	�‰�Q‹ß	‰��E˜2˜xÓ	(ß	‰��A�q˜!Ó	ß	‰��B˜ Ñ(Ó	)ð	 ð €I÷Q MÕLûô$  ×-Ñ-ó 	Ø Ó%Ü ×1Ò1°$Ô7Ø  EÓ)Ø$(�MÜÐPÔQÞØ˜‘
�Ø˜2“:Ø�GÜÐNÐPU×VÑVà�ûð	úò1 s>   Æ-A8M È%6MÉBM Í
M	ÍM ÍOÍ*/O Î O Î>O Ï OFz0.0.21z0.0.22z0.0.23c                 ó¾  ^^	^
• U R                   u  m	nm
T
T-  m
[        (       a  T	T-  S:”  a  [        XUTU5      $ [        U	U
U4S jXU45      u  pnUb¡  SU R                   S   S-  -
  n[        R
                  " U R                   S   U R                   S   U R                   S   U-   /U R                  U R                  S9nXGS S 2S S 2S UR                   S   24'   US S 2S S 2S UR                   S   24   n[        R                  R                  XX$S9nUR                  S5      R                  T	TST
5      R                  SS	SS
5      R                  T	STT
-  5      nU$ )Niÿÿ  c                 ó²   >• U R                  S5      R                  TSTT5      R                  SSSS5      R                  TT-  ST5      R                  5       $ rx   rz   r   s    €€€r   rƒ   Ú$attention_xformers.<locals>.<lambda>%  r…   r   rš   r	   r   r8   r*   )Ú	attn_biasr7   ry   )r+   ÚBROKEN_XFORMERSÚattention_pytorchr‰   r"   Úemptyr$   r9   ÚxformersrR   Úmemory_efficient_attentionr{   r|   r}   )r�   r‘   r’   r‚   r“   r”   ÚpadÚmask_outr—   r€   r�   s      `     @@r   Úattention_xformersrä     sZ  ú€ Ø—W‘W�N€A€qˆ(Ø�Ñ€Hß‚Øˆu‰9�uÓÜ$ Q¨1¨e°TÓ:Ð:äö	ð
 
ˆqˆ	ó�G€Aˆ!ð ÑØ�!—'‘'˜!‘*˜q‘.Ñ ˆÜ—;’; §¡¨¡
¨A¯G©G°A©J¸¿¹À¹
ÀSÑ8HÐIÐQR×QXÑQXÐab×aiÑaiÑjˆØ*.’’A�˜Ÿ
™
 2™�Ð&Ñ'Øšš1˜o˜tŸz™z¨"™~˜oÐ-Ñ.ˆä
�,‰,×
1Ñ
1°!¸Ð
1Ð
J€Cð 	�‰�aÓß	‰��E˜2˜xÓ	(ß	‰��A�q˜!Ó	ß	‰��B˜ Ñ(Ó	)ð	 ð €Jr   c           	      ó  ^^^• U R                   u  mnmTT-  m[        UUU4S jXU45      u  pn[        R                  R                  R                  XX$SSS9nUR                  SS5      R                  TSTT-  5      nU$ )Nc                 óL   >• U R                  TSTT5      R                  SS5      $ )Nr*   r	   r7   )Úviewr§   r   s    €€€r   rƒ   Ú#attention_pytorch.<locals>.<lambda>A  s!   ø€ �!—&‘&˜˜B  xÓ0×:Ñ:¸1¸aÔ@r   rZ   F)Ú	attn_maskÚ	dropout_pÚ	is_causalr	   r7   r*   )r+   r‰   r"   r   Ú
functionalÚscaled_dot_product_attentionr§   r|   )	r�   r‘   r’   r‚   r“   r”   r—   r€   r�   s	      `   @@r   rÞ   rÞ   =  s‰   ú€ Ø—W‘W�N€A€qˆ(Ø�Ñ€HÜÞ@Ø	
ˆqˆ	ó�G€Aˆ!ô
 �(‰(×
Ñ
×
:Ñ
:¸1ÀÐ^aÐmrÐ
:Ð
s€Cà�‰�a˜Ó×#Ñ# A r¨5°8Ñ+;Ó<ð ð €Jr   zUsing xformers cross attentionzUsing pytorch cross attentionz,Using split optimization for cross attentionzuUsing sub quadratic optimization for cross attention, if you have memory or speed issues try using: --attention-splitc                 óÂ   • U(       a&  [         R                  " 5       (       a  [        $ [        $ U [        R
                  " S5      :X  a  [        $ U(       a  [        $ [        $ )NÚcpu)	r   Úpytorch_attention_enabledrÞ   r˜   r"   r9   rµ   Úoptimized_attention_maskedÚoptimized_attention)r9   r“   Úsmall_inputs      r   Úoptimized_attention_for_devicerô   ^  sF   € ÞÜ×5Ò5×7Ñ7Ü$Ð$ä"Ð"à”—’˜eÓ$Ó$Ü!Ð!æÜ)Ð)äÐr   c                   óD   ^ • \ rS rSrSSSSSS\4U 4S jjrS	S jrSrU =r$ )
ÚCrossAttentionin  Nrš   r¹   rZ   c	           	      óX  >• [         T
U ]  5         XC-  n	[        X!5      nX0l        X@l        UR                  XSXgS9U l        UR                  X)SXgS9U l        UR                  X)SXgS9U l        [        R                  " UR                  X‘XgS9[        R                  " U5      5      U l        g )NF)Úbiasr$   r9   r8   )r:   r;   r    r‚   r�   r<   Úto_qÚto_kÚto_vr   r]   r_   Úto_out)r>   Ú	query_dimÚcontext_dimr‚   r�   rc   r$   r9   rA   rd   rB   s             €r   r;   ÚCrossAttention.__init__o  s§   ø€ Ü‰ÑÔØÑ$ˆ	Ü˜kÓ5ˆàŒ
Ø Œà×%Ñ% iÀÈeÐ%ÐcˆŒ	Ø×%Ñ% kÀ5ÐPUÐ%ÐeˆŒ	Ø×%Ñ% kÀ5ÐPUÐ%ÐeˆŒ	ä—m’m J×$5Ñ$5°iÐRWÐ$5Ð$gÔik×isÒisÐt{Ói|Ó}ˆ�r   c                 ó.  • U R                  U5      n[        X!5      nU R                  U5      nUb  U R                  U5      nAOU R                  U5      nUc  [	        XVXpR
                  5      nO[        XVXpR
                  U5      nU R                  U5      $ r   )rù   r    rú   rû   rò   r‚   rñ   rü   )	r>   rI   Úcontextr«   r“   r�   r‘   r’   r—   s	            r   rK   ÚCrossAttention.forward}  s‚   € Ø�I‰I�a‹LˆÜ˜'Ó%ˆØ�I‰I�gÓˆØÑØ—	‘	˜%Ó ˆAÙà—	‘	˜'Ó"ˆAà‰<Ü% a¨A¯z©zÓ:‰Cä,¨Q°1·j±jÀ$ÓGˆCØ�{‰{˜3ÓÐr   )r�   r‚   rú   rü   rù   rû   )NNNrM   rU   s   @r   rö   rö   n  s&   ø† Ø.2¸!ÀbÐRTÐ\`ÐimÐz}÷ ~÷ ò  r   rö   c                   ó`   ^ • \ rS rSrSSSSSSSSSSS\4U 4S jjrS0 4S jrS0 4S jrS	rU =r	$ )
ÚBasicTransformerBlockiŽ  rZ   NTFc                 óh  >• [         TU ]  5         U=(       d    U	S LU l        U	c  Un	X‘:H  U l        U R                  (       a&  UR	                  XUS9U l        [        XXFXÞUS9U l        X l        [        X’X4U R                  (       a  UOS XÞUS9U l	        [        X‘XFXÞUS9U l
        U(       a  U(       a  [        eS U l        O2S nU(       d  Un[        U	UX#XMXïS9U l        UR	                  X�US9U l        UR	                  X�US9U l        UR	                  X�US9U l        Xpl        X l        X0l        XÀl        g )Nr8   )r@   rc   rb   r$   r9   rA   )rý   r‚   r�   rc   rþ   r$   r9   rA   )rý   rþ   r‚   r�   rc   r$   r9   rA   )r:   r;   Úff_inÚis_resÚ	LayerNormÚnorm_inrW   Údisable_self_attnrö   Úattn1ÚffÚ
ValueErrorÚattn2Únorm2Únorm1Únorm3r
   Ún_headsÚd_headÚswitch_temporal_ca_to_sa)r>   r0   r  r  rc   rþ   Úgated_ffr
   r  rd   r
  Údisable_temporal_crossattentionr  r$   r9   rA   Úcontext_dim_attn2rB   s                    €r   r;   ÚBasicTransformerBlock.__init__�  sN  ø€ ä‰ÑÔà×3˜i¨tÐ3ˆŒ
ØÑØˆIàÑ&ˆŒà�:�:Ø%×/Ñ/°ÈÐ/ÐPˆDŒLÜ$ SÀWÐbgð  DNñ  OˆDŒJà!2ÔÜ#¨iÐQWØ9=×9O×9O©+ÐUYÐafð  CMñNˆŒ
ä˜i¸gÐ[`ð  }Gñ  HˆŒæ*Þ'Ü Ð à!�•
à $ÐÞ+Ø$/Ð!ä'°)ÐIZØ&-ÈÐekñDˆDŒJà#×-Ñ-¨iÈVÐ-ÐTˆDŒJà×)Ñ)¨)ÈÐ)ÐPˆŒ
Ø×)Ñ)¨)ÈÐ)ÐPˆŒ
Ø$ŒØŒØŒØ(@Õ%r   c                 óf   • [        U R                  XU4U R                  5       U R                   5      $ r   )r
   Ú_forwardÚ
parameters)r>   rI   r  Útransformer_optionss       r   rK   ÚBasicTransformerBlock.forward·  s+   € Ü˜$Ÿ-™-¨!Ð6IÐ)JÈDÏOÉOÓL]Ð_c×_nÑ_nÓoÐor   c                 ó  • 0 nUR                  SS 5      nUR                  SS5      n0 n0 nU H!  n	U	S:X  a  X9   nM  U	S:X  a  X9   nM  X9   XI'   M#     U R                  US'   U R                  US'   U R                  (       a7  Un
U R                  U R	                  U5      5      nU R
                  (       a  X-  nU R                  U5      nU R                  (       a  UnOS nS nSU;   a!  US   nUc  UnUnU H  nU" X¼XÔ5      u  p¼nM     Ub  US   US	   U4nOS nUR                  S
0 5      nUnUU;  a  UnUU;   a€  Uc  UnUnU R                  R                  U5      nU R                  R                  U5      nU R                  R                  U5      nUU   " X¼XÔ5      nU R                  R                  U5      nOU R                  X¼US9nSU;   a  US   nU H  nU" X´5      nM     X-  nSU;   a  US   nU H  nU" X5      nM     U R                  Gb  U R                  U5      nU R                  (       a  UnOUnS nSU;   a  US   nUnU H  nU" UUUU5      u  nnnM     UR                  S0 5      nUnUU;  a  UnUU;   a€  Uc  UnU R                  R                  U5      nU R                  R                  U5      nU R                  R                  U5      nUU   " UUUU5      nU R                  R                  U5      nOU R                  UUUS9nSU;   a  US   nU H  nU" X´5      nM     X-  nU R
                  (       a  Un
U R!                  U R#                  U5      5      nU R
                  (       a  UW
-  nU$ )NÚblockÚblock_indexr   ÚpatchesÚpatches_replacer  r�   Úattn1_patchr	   r  )r  r«   Úattn1_output_patchÚmiddle_patchÚattn2_patchr  Úattn2_output_patch)Úgetr  r  r  r	  r  r  r
  r  rù   rú   rû   rü   r  r  r  r  r  )r>   rI   r  r  Úextra_optionsr  r   Útransformer_patchesÚtransformer_patches_replacer‘   Úx_skipÚnÚcontext_attn1Úvalue_attn1ÚpatchÚpÚtransformer_blockÚattn1_replace_patchÚblock_attn1Úcontext_attn2Úvalue_attn2Úattn2_replace_patchÚblock_attn2s                          r   r  ÚBasicTransformerBlock._forwardº  sÉ  € ØˆØ#×'Ñ'¨°Ó6ˆØ)×-Ñ-¨m¸QÓ?ˆØ ÐØ&(Ð#ã$ˆAØ�I‹~Ø&9Ñ&<Ò#ØÐ'Ó'Ø.AÑ.DÒ+à#6Ñ#9�Ó ñ %ð $(§<¡<ˆ�iÑ Ø$(§K¡Kˆ�jÑ!à�:�:ØˆFØ—
‘
˜4Ÿ<™<¨›?Ó+ˆAØ�{�{Ø‘�à�J‰J�q‹MˆØ×!×!Ø#‰Mà ˆMØˆàÐ/Ó/Ø'¨Ñ6ˆEØÑ$Ø !�Ø'ˆKÛ�Ù01°!ÀKÓ0_Ñ-�¢+ñ ð ÑØ!& q¡¨5°©8°[Ð AÑà $ÐØ9×=Ñ=¸gÀrÓJÐØ'ˆØÐ1Ó1ØˆKàÐ-Ó-ØÑ$Ø !�Ø�Ø—
‘
—‘ Ó"ˆAØ ŸJ™JŸO™O¨MÓ:ˆMØŸ*™*Ÿ/™/¨+Ó6ˆKØ# KÒ0°À;Ó^ˆAØ—
‘
×!Ñ! !Ó$‰Aà—
‘
˜1¸;�
ÐGˆAàÐ#6Ó6Ø'Ð(<Ñ=ˆEÛ�Ù�aÓ'’ñ ð 	
‰ˆØÐ0Ó0Ø'¨Ñ7ˆEÛ�Ù�aÓ'’ñ ð �:‰:Ò!Ø—
‘
˜1“ˆAØ×,×,Ø !‘à '�ØˆKØÐ 3Ó3Ø+¨MÑ:�Ø+�Û�AÙ45°a¸ÈÐUbÓ4cÑ1�A�}¢kñ ð #>×"AÑ"AÀ'È2Ó"NÐØ+ˆKØÐ"5Ó5Ø#�àÐ1Ó1ØÑ&Ø"/�KØ—J‘J—O‘O AÓ&�Ø $§
¡
§¡°Ó >�Ø"Ÿj™jŸo™o¨kÓ:�Ø'¨Ò4°Q¸À{ÐTaÓb�Ø—J‘J×%Ñ% aÓ(‘à—J‘J˜q¨-¸{�JÐK�àÐ#6Ó6Ø'Ð(<Ñ=ˆEÛ�Ù�aÓ'’ñ ð 	
‰ˆØ�;�;ØˆFØ�G‰G�D—J‘J˜q“MÓ"ˆØ�;�;Ø�‰KˆAàˆr   )r  r  r
   r  r
  r  r  r  r  r  r  r  r	  r  )
rN   rO   rP   rQ   rR   r;   rK   r  rS   rT   rU   s   @r   r  r  Ž  sX   ø† Ø57ÀTÐTXÐeiÐqvð  CGØ#(È%ÐjoÐw{ð  EIð  VY÷&AðP "&¸2ô pð #'¸B÷ pò pr   r  c            	       óP   ^ • \ rS rSrSrSSSSSSSS\4	U 4S jjrS0 4S	 jrS
rU =r	$ )ÚSpatialTransformeri-  zè
Transformer block for image-like data.
First, project the input (aka embedding)
and reshape to b, t, d.
Then apply standard transformer action.
Finally, reshape to image
NEW: use_linear for more efficiency instead of the 1x1 convs
r	   rZ   NFTc                 ó.  >• [         TU ]  5         [        U5      (       a  [        U[        5      (       d  U/U-  nXl        X#-  nUR                  SUSSX«S9U l        U(       d  UR                  UUSSSX«S9U l	        OUR                  XX«S9U l	        [        R                  " [        U5       Vs/ s H  n[        XÒX5Xn   XyX«US	9
PM     sn5      U l        U(       d  UR                  XÑSSSX«S9U l        OUR                  XX«S9U l        X€l        g s  snf )
Nrk   rl   Trm   r	   r   )Úkernel_sizeÚstrideÚpaddingr$   r9   r8   )rc   rþ   r
  r
   r$   r9   rA   )r:   r;   r   Ú
isinstanceÚlistrs   rr   ÚnormÚConv2dÚproj_inr<   r   Ú
ModuleListrÃ   r  Útransformer_blocksÚproj_outÚ
use_linear)r>   rs   r  r  Údepthrc   rþ   r
  rH  r    r$   r9   rA   rd   r   rB   s                  €r   r;   ÚSpatialTransformer.__init__6  sD  ø€ ô 	‰ÑÔÜ�+×Ñ¤z°+¼t×'DÑ'DØ&˜-¨%Ñ/ˆKØ&ÔØÑ$ˆ	Ø×(Ñ(°BÀ[ÐVZÐcgÐotÐ(ð  EˆŒ	ÞØ%×,Ñ,¨[Ø%.Ø12Ø,-Ø-.°eð	 -ð LˆD�Lð &×,Ñ,¨[È5Ð,Ð`ˆDŒLä"$§-¢-ô ˜uœó'â%�Aô # 9°vÐ\gÑ\jØ5FÐinð  KUôVá%ñ'ó#
ˆÔö
 Ø&×-Ñ-¨iØ>?Ø9:Ø:;À5ð .ð YˆD�Mð
 '×-Ñ-¨kÈEÐ-ÐaˆDŒMØ$�ùò's   Â0Dc                 óx  • [        U[        5      (       d  U/[        U R                  5      -  nUR                  u  pEpgUnU R                  U5      nU R                  (       d  U R                  U5      n[        US5      R                  5       nU R                  (       a  U R                  U5      n[        U R                  5       H  u  pšX“S'   U
" XU	   US9nM     U R                  (       a  U R                  U5      n[        USXgS9R                  5       nU R                  (       d  U R                  U5      nX-   $ )Núb c h w -> b (h w) cr   ©r  r  úb (h w) c -> b c h w©r‡   Úw)r@  rA  rÅ   rF  r+   rB  rH  rD  r   r~   Ú	enumeraterG  )r>   rI   r  r  r€   Úcr‡   rP  Úx_inrÓ   r  s              r   rK   ÚSpatialTransformer.forwardW  sþ   € ä˜'¤4×(Ñ(Ø�i¤# d×&=Ñ&=Ó">Ñ>ˆGØ—W‘W‰
ˆˆaØˆØ�I‰I�a‹LˆØ��Ø—‘˜Q“ˆAÜ�aÐ/Ó0×;Ñ;Ó=ˆØ�?�?Ø—‘˜Q“ˆAÜ! $×"9Ñ"9Ö:‰HˆAØ12 Ñ.Ù�a¨¡ÐATÑUŠAñ ;ð �?�?Ø—‘˜aÓ ˆAÜ�aÐ/°1Ñ:×EÑEÓGˆØ��Ø—‘˜aÓ ˆAØ‰xˆr   )rs   rB  rD  rG  rF  rH  )
rN   rO   rP   rQ   Ú__doc__rR   r;   rK   rS   rT   rU   s   @r   r;  r;  -  s6   ø† ñð  "°$Ø#(°UØ $¨D¸È#÷%ðB "&¸2÷ ò r   r;  c                   ó  ^ • \ rS rSrSSSSSSSSSSSSSSSSS\4S	\S
\S\4U 4S jjjrSSSS0 4S\	R                  S\\	R                     S\\	R                     S\\   S\\	R                     S\	R                  4S jjrSrU =r$ )ÚSpatialVideoTransformerin  r	   rZ   FNÚfixedg      à?i'  Úmerge_strategyÚmerge_factorÚmax_time_embed_periodc                 ó¾  >• [         TU ]  UUUUUUUUUUUUS9  Xðl        X@l        UU l        UnUn[        UU-  5      nX#-  nU(       a  Un[        R                  " [        U R                  5       Vs/ s H  n[        UUUUUUUUUUUUUS9PM     sn5      U l
        [        U R                  5      [        U R                  5      :X  d   eX€l        Xl        U R                  S-  n[        R                  " UR!                  U R                  UUUS9[        R"                  " 5       UR!                  UU R                  UUS95      U l        ['        XºS9U l        g s  snf )N)	rI  rc   r    rþ   rH  r
  r$   r9   rA   )
rc   rþ   r
   r  rd   r
  r  r$   r9   rA   rY   r8   )ÚalpharY  )r:   r;   Ú
time_depthrI  r[  r\   r   rE  rÃ   r  Ú
time_stackrÅ   rF  Úuse_spatial_contextrs   r]   r<   ÚSiLUÚtime_pos_embedr   Ú
time_mixer)r>   rs   r  r  rI  rc   rH  rþ   r`  Ú	timestepsrY  rZ  Útime_context_dimr  r
   r^  r
  r  r[  r$   r9   rA   Útime_mix_d_headÚn_time_mix_headsÚtime_mix_inner_dimrd   r”   Útime_embed_dimrB   s                               €r   r;   Ú SpatialVideoTransformer.__init__o  sŠ  ø€ ô, 	‰ÑØØØØØØ%Ø#Ø!Ø/Ø °:ð 	ñ 	
ð %ŒØŒ
Ø%:ˆÔ"à ˆØ"Ðä  Ð3CÑ!CÓDÐàÑ$ˆ	ÞØ*ÐäŸ-š-ô  ˜tŸz™zÔ*óò +�Aô &ØØ$Ø#Ø#Ø 0à)ØØ0Ø&7Ø4SØ¨¸:ôñ +ñó
ˆŒô( �4—?‘?Ó#¤s¨4×+BÑ+BÓ'CÓCÐCÐCà#6Ô Ø&Ôà×)Ñ)¨AÑ-ˆÜ ŸmšmØ×Ñ˜d×.Ñ.°ÀeÐTZÐÐ[Ü�GŠG‹IØ×Ñ˜n¨d×.>Ñ.>ÀeÐTZÐÐ[ó
ˆÔô 'Øñ
ˆ�ùò?s   Á3ErI   r  Útime_contextrd  Úimage_only_indicatorÚreturnc           
      ó$  • UR                   u    pxn	Un
S n[        U5      (       a  UnU R                  (       a?  UR                  S:X  d   SUR                   35       eUc  UnUS S U2   n[	        USX‰-  S9nO=Ub:  U R                  (       d)  [	        USX‰-  S9nUR                  S:X  a  [        US5      nU R                  U5      nU R                  (       d  U R                  U5      n[        US5      nU R                  (       a  U R                  U5      n[        R                  " XAR                  S9n[	        US	UR                   S
   U-  S9n[        US5      n[        XÐR                  SU R                  S9R                  UR                   5      nU R#                  U5      nUS S 2S S S 24   n[%        ['        U R(                  U R*                  5      5       Ha  u  nu  nnUUS'   U" UUUS9nUnUU-   nUR                   u  nnn[        USUS9nU" UUS9n[        USUUU-  UUS9nU R-                  UUUS9nMc     U R                  (       a  U R/                  U5      n[        USX‰S9nU R                  (       d  U R/                  U5      nX-   nU$ )Nry   z.n dims of spatial context should be 3 but are zb ... -> (b n) ...)r-  r7   zb c -> b 1 crL  )r9   zt -> b tr   )r€   zb t -> (b t)F)Úrepeat_onlyÚ
max_periodr   rM  z(b t) s c -> (b s) t cr&   )r  z(b s) t c -> (b t) s c)Úsr€   rR  r'   )Ú	x_spatialÚ
x_temporalrl  rN  rO  )r+   r   r`  Úndimr   r   rB  rH  rD  r"   Úaranger9   r   rs   r[  r�   r$   rb  rQ  ÚziprF  r_  rc  rG  )r>   rI   r  rk  rd  rl  r  r”   r‡   rP  rS  Úspatial_contextÚtime_context_first_timestepÚ
num_framesÚt_embÚembÚit_r  Ú	mix_blockÚx_mixÚBÚSÚCr—   s                           r   rK   ÚSpatialVideoTransformer.forwardÂ  s›  € ð —W‘W‰
ˆˆ1�ØˆØˆÜ�'�?‰?Ø%ˆOà×#×#à—‘ Ó!ðOà?ÀÇÁ¸~ÐNóOØ!ð Ñ#Ø&�Ø*6±{¸°{Ñ*CÐ'Ü!Ø+Ð-AÀQÁUñ‰Lð Ñ%¨d×.F×.FÜ! ,Ð0DÈÉÑNˆLØ× Ñ  AÓ%Ü(¨°~ÓF�à�I‰I�a‹LˆØ��Ø—‘˜Q“ˆAÜ�aÐ/Ó0ˆØ�?�?Ø—‘˜Q“ˆAä—\’\ )·H±HÑ=ˆ
Ü˜J¨
°a·g±g¸a±jÀIÑ6MÑNˆ
Ü˜z¨>Ó:ˆ
Ü" :×/?Ñ/?ÈUÐ_c×_yÑ_yÑz×}Ñ}Ð~÷  Fñ  Fó  GˆØ×!Ñ! %Ó(ˆØ’!�Tš1�*‰oˆä'0Ü�×'Ñ'¨¯©Ó9ö(
Ñ#ˆCÑ#�%˜ð 25Ð Ñ.ÙØØ'Ø$7ñˆAð ˆEØ˜C‘KˆEà—k‘k‰GˆAˆq�!Ü˜eÐ%=ÀÑKˆEÙ˜e¨\Ñ:ˆEÜØÐ/°1¸¸Y¹È!ÈyñˆEð —‘¨!¸ÐTh�ÐiŠAñ)(
ð, �?�?Ø—‘˜aÓ ˆAÜ�aÐ/°1Ñ:ˆØ��Ø—‘˜aÓ ˆAØ‰hˆØˆ
r   )rI  rs   r[  r^  rc  rb  r_  r`  )rN   rO   rP   rQ   rR   Ústrr‹   r\   r;   r"   ÚTensorr   rK   rS   rT   rU   s   @r   rW  rW  n  së   ø† ð ØØØØ!ØØ%Ø!ØØØØØØ(-Ø%*Ø˜4¨Cñ)Q
ð ðQ
ð ðQ
ð&  #÷'Q
ð Q
ðl +/Ø/3Ø#'Ø7;ØñIà�<‰<ðIð ˜%Ÿ,™,Ñ'ðIð ˜uŸ|™|Ñ,ð	Ið
 ˜C‘=ðIð ' u§|¡|Ñ4ðIð 
�‰÷Ió Ir   rW  )NNr   )FF)<r,   r"   Útorch.nn.functionalr   rì   rG   r   Úeinopsr   r   Útypingr   r   Údiffusionmodules.utilr
   r   r   Úsub_quadratic_attentionr   Úldm_patched.modulesr   Úxformers_enabledrà   Úxformers.opsÚldm_patched.modules.args_parserr   Úldm_patched.modules.opsÚldm_patchedÚmodulesrR   Údisable_weight_initÚdisable_attention_upcastrÈ   rŠ   r   r   r    r(   r2   ÚModuler4   rW   rt   r˜   rµ   rØ   rÝ   Ú__version__Úx_versÚ
startswithrä   rÞ   rò   rð   rñ   rô   rö   r  r;  rW  r   r   r   Ú<module>r—     sç  ðÛ Û ß Ð ß ß $ß  ç OÑ OÝ Då 0à×$Ò$×&Ñ&ÛÛå 0Û Ø×Ñ×Ñ×1Ñ1€ð × × Ù	Ð
,Ô-Ø�Oà€Oòò*òò%òô ˆB�I‰Iô  ô�"—)‘)ô ô&zô*ôZ3ôjZðx €ð	Ø×!Ñ!€Fà×'Ñ'¨Ó1×o°V×5FÑ5FÀxÓ5P×oÐTZ×TeÑTeÐfnÓTo€Oôô@ð &Ð à×$Ò$×&Ñ&Ù	Ð
*Ô+Ø,ÑØ×/Ò/×1Ñ1Ù	Ð
)Ô*Ø+Ñà××ÙÐ<Ô=Ø-Ñáð  Fô  	GØ0Ðà0Ð ôô  �R—Y‘Yô  ô@\˜BŸI™Iô \ô~>˜Ÿ™ô >ôB]Ð0õ ]øði
	Ùús   Ã6AH	 È	H