ó
    USh�&  ã                   óÔ  • S SK r S SKJr  S SKJr  S rS r " S S\R                  5      r " S S\R                  5      r	 " S	 S
\R                  5      r
 " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      rg)é    N)ÚOrderedDictc                 óÆ   • U S:X  a  [         R                  " U0 UD6$ U S:X  a  [         R                  " U0 UD6$ U S:X  a  [         R                  " U0 UD6$ [	        SU  35      e)z,
Create a 1D, 2D, or 3D convolution module.
é   é   é   úunsupported dimensions: )ÚnnÚConv1dÚConv2dÚConv3dÚ
ValueError©ÚdimsÚargsÚkwargss      Ú</home/gothic/public_html/Fooocus/ldm_patched/t2ia/adapter.pyÚconv_ndr      sh   € ð ˆqƒyÜ�yŠy˜$Ð) &Ñ)Ð)Ø	�‹Ü�yŠy˜$Ð) &Ñ)Ð)Ø	�‹Ü�yŠy˜$Ð) &Ñ)Ð)Ü
Ð/°¨vÐ6Ó
7Ð7ó    c                 óÆ   • U S:X  a  [         R                  " U0 UD6$ U S:X  a  [         R                  " U0 UD6$ U S:X  a  [         R                  " U0 UD6$ [	        SU  35      e)z0
Create a 1D, 2D, or 3D average pooling module.
r   r   r   r   )r	   Ú	AvgPool1dÚ	AvgPool2dÚ	AvgPool3dr   r   s      r   Úavg_pool_ndr      sh   € ð ˆqƒyÜ�|Š|˜TÐ, VÑ,Ð,Ø	�‹Ü�|Š|˜TÐ, VÑ,Ð,Ø	�‹Ü�|Š|˜TÐ, VÑ,Ð,Ü
Ð/°¨vÐ6Ó
7Ð7r   c                   ó6   ^ • \ rS rSrSrSU 4S jjrS rSrU =r$ )Ú
Downsampleé!   a,  
A downsampling layer with an optional convolution.
:param channels: channels in the inputs and outputs.
:param use_conv: a bool determining if a convolution is applied.
:param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then
             downsampling occurs in the inner-two dimensions.
c           	      ó*  >• [         TU ]  5         Xl        U=(       d    UU l        X l        X0l        US:w  a  SOSnU(       a&  [        X0R                  U R                  SXeS9U l        g U R                  U R                  :X  d   e[        X6US9U l        g )Nr   r   )r   r   r   )ÚstrideÚpadding)Úkernel_sizer   )	ÚsuperÚ__init__ÚchannelsÚout_channelsÚuse_convr   r   Úopr   )Úselfr#   r%   r   r$   r   r   Ú	__class__s          €r   r"   ÚDownsample.__init__*   s�   ø€ Ü‰ÑÔØ ŒØ(×4¨HˆÔØ ŒØŒ	Ø˜a“i‘ YˆÞÜØ—m‘m T×%6Ñ%6¸À&ñˆD�Gð —=‘= D×$5Ñ$5Ó5Ð5Ð5Ü! $À6ÑJˆD�Gr   c                 óð   • UR                   S   U R                  :X  d   eU R                  (       d4  UR                   S   S-  UR                   S   S-  /nX R                  l        U R                  U5      nU$ )Nr   r   r   )Úshaper#   r%   r&   r   )r'   Úxr   s      r   ÚforwardÚDownsample.forward9   s`   € Ø�w‰w�q‰z˜TŸ]™]Ó*Ð*Ð*Ø�}�}Ø—w‘w˜q‘z A‘~ q§w¡w¨q¡z°A¡~Ð6ˆGØ%�G‰GŒOà�G‰G�A‹JˆØˆr   )r#   r   r&   r$   r%   )r   Nr   )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r"   r-   Ú__static_attributes__Ú__classcell__©r(   s   @r   r   r   !   s   ø† ñ÷K÷ð r   r   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )ÚResnetBlockéC   c                 óÜ  >• [         TU ]  5         US-  nX:w  d  US:X  a  [        R                  " XUSU5      U l        OS U l        [        R                  " X"SSS5      U l        [        R                  " 5       U l        [        R                  " X"USU5      U l        US:X  a  [        R                  " XUSU5      U l	        OS U l	        X0l
        U R                  S:X  a  [        XS9U l        g g )Nr   Fr   r   T©r%   )r!   r"   r	   r   Úin_convÚblock1ÚReLUÚactÚblock2ÚskepÚdownr   Údown_opt)	r'   Úin_cÚout_crB   ÚksizeÚskr%   Úpsr(   s	           €r   r"   ÚResnetBlock.__init__D   sÀ   ø€ Ü‰ÑÔØ�a‰ZˆØ‹=˜B %›KÜŸ9š9 T°%¸¸BÓ?ˆD�Lð  ˆDŒLÜ—i’i ¨a°°AÓ6ˆŒÜ—7’7“9ˆŒÜ—i’i ¨e°Q¸Ó;ˆŒØ�‹;ÜŸ	š	 $¨u°a¸Ó<ˆD�IàˆDŒIàŒ	Ø�9‰9˜ÓÜ& tÑ?ˆD�Mð r   c                 ó.  • U R                   S:X  a  U R                  U5      nU R                  b  U R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  b  X R                  U5      -   $ X!-   $ ©NT)rB   rC   r<   r=   r?   r@   rA   ©r'   r,   Úhs      r   r-   ÚResnetBlock.forwardX   s|   € Ø�9‰9˜ÓØ—‘˜aÓ ˆAØ�<‰<Ñ#Ø—‘˜Q“ˆAà�K‰K˜‹NˆØ�H‰H�Q‹KˆØ�K‰K˜‹NˆØ�9‰9Ñ Ø—y‘y “|Ñ#Ð#à‘5ˆLr   )r?   r=   r@   rB   rC   r<   rA   )r   FT©r/   r0   r1   r2   r"   r-   r4   r5   r6   s   @r   r8   r8   C   s   ø† ÷@÷(ð r   r8   c                   óD   ^ • \ rS rSr/ SQSSSSSS4U 4S jjrS rS	rU =r$ )
ÚAdapterég   ©i@  i€  é   rT   r   é@   FTc                 ó^  >• [         [        U ]  5         SU l        / n/ SQn	Xpl        U R                  (       a  SU l        S/nS/n	X0R                  U R                  -  -  U l        [        R                  " U R                  5      U l        Xl	        X l
        / U l        [        [        U5      5       H³  n
[        U5       H¡  nX©;   a5  US:X  a/  U R                  R                  [        XS-
     X   SXEUS95        M=  X¨;   a5  US:X  a/  U R                  R                  [        XS-
     X   S	XEUS95        Mw  U R                  R                  [        X   X   S	XEUS95        M£     Mµ     [        R                   " U R                  5      U l        [        R"                  " X1S   S
SS5      U l        g )Né   )r   r   r   é   r   r   r   T)rB   rF   rG   r%   Fr   )r!   rQ   r"   Úunshuffle_amountÚxlÚinput_channelsr	   ÚPixelUnshuffleÚ	unshuffler#   Únums_rbÚbodyÚrangeÚlenÚappendr8   Ú
ModuleListr   Úconv_in)r'   r#   r^   ÚcinrF   rG   r%   rZ   Úresblock_no_downsampleÚresblock_downsampleÚiÚjr(   s               €r   r"   ÚAdapter.__init__h   s{  ø€ ÜŒg�tÑ%Ô'Ø !ˆÔØ!#ÐÚ'ÐØŒØ�7�7Ø$&ˆDÔ!Ø&' SÐ"Ø#$ #Ðà!×&;Ñ&;¸d×>SÑ>SÑ&SÑTˆÔÜ×*Ò*¨4×+@Ñ+@ÓAˆŒØ ŒØŒØˆŒ	Ü”s˜8“}Ö%ˆAÜ˜7–^�ØÓ,°1¸³6Ø—I‘I×$Ñ$Ü# H°©U¡O°X±[ÀtÐSXÐjrÑsöuàÓ1¸¸Q»Ø—I‘I×$Ñ$Ü# H°©U¡O°X±[ÀuÐTYÐksÑtövð —I‘I×$Ñ$Ü# H¡K°±À5ÐPUÐgoÑpöró $ñ &ô —M’M $§)¡)Ó,ˆŒ	Ü—y’y ¨q¡k°1°a¸Ó;ˆ�r   c                 óF  • U R                  U5      n/ nU R                  U5      n[        [        U R                  5      5       HÚ  n[        U R
                  5       H)  nX0R
                  -  U-   nU R                  U   " U5      nM+     U R                  (       aQ  UR                  S 5        US:X  a"  UR                  S 5        UR                  S 5        US:X  a  UR                  S 5        O"UR                  S 5        UR                  S 5        UR                  U5        MÜ     U$ )Nr   r   )	r]   rd   r`   ra   r#   r^   r_   rZ   rb   )r'   r,   Úfeaturesrh   ri   Úidxs         r   r-   ÚAdapter.forward†   sã   € à�N‰N˜1ÓˆàˆØ�L‰L˜‹OˆÜ”s˜4Ÿ=™=Ó)Ö*ˆAÜ˜4Ÿ<™<Ö(�ØŸ,™,Ñ&¨Ñ*�Ø—I‘I˜c’N 1Ó%’ñ )ð �w�wØ—‘ Ô%Ø˜“6Ø—O‘O DÔ)Ø—O‘O DÔ)Ø˜“6Ø—O‘O DÔ)øà—‘ Ô%Ø—‘ Ô%Ø�O‰O˜AÖñ +ð  ˆr   )r_   r#   rd   r[   r^   r]   rY   rZ   rO   r6   s   @r   rQ   rQ   g   s%   ø† Ú 6ÀÀrÐQRÐW\ÐgkÐpt÷ <÷<ð r   rQ   c                   óH   ^ • \ rS rSrSrS\R                  4U 4S jjrSrU =r	$ )Ú	LayerNorméŸ   z*Subclass torch's LayerNorm to handle fp16.r,   c                 ó–   >• UR                   n[        TU ]	  UR                  [        R
                  5      5      nUR                  U5      $ ©N)Údtyper!   r-   ÚtypeÚtorchÚfloat32)r'   r,   Ú	orig_typeÚretr(   s       €r   r-   ÚLayerNorm.forward¢   s6   ø€ Ø—G‘Gˆ	Ü‰g‰o˜aŸf™f¤U§]¡]Ó3Ó4ˆØ�x‰x˜	Ó"Ð"r   © )
r/   r0   r1   r2   r3   rv   ÚTensorr-   r4   r5   r6   s   @r   rp   rp   Ÿ   s   ø† Ù4ð#˜Ÿ™÷ #õ #r   rp   c                   ó6   • \ rS rSrS\R
                  4S jrSrg)Ú	QuickGELUé¨   r,   c                 ó:   • U[         R                  " SU-  5      -  $ )Ng¬Zd;û?)rv   Úsigmoid©r'   r,   s     r   r-   ÚQuickGELU.forwardª   s   € Ø”5—=’= ¨¡Ó+Ñ+Ð+r   r{   N)r/   r0   r1   r2   rv   r|   r-   r4   r{   r   r   r~   r~   ¨   s   † ð,˜Ÿ™÷ ,r   r~   c                   ó”   ^ • \ rS rSrS
S\S\S\R                  4U 4S jjjrS\R                  4S jrS\R                  4S jr	S	r
U =r$ )ÚResidualAttentionBlocké®   Úd_modelÚn_headÚ	attn_maskc                 ól  >• [         TU ]  5         [        R                  " X5      U l        [        U5      U l        [        R                  " [        S[        R                  " XS-  5      4S[        5       4S[        R                  " US-  U5      4/5      5      U l        [        U5      U l        X0l        g )NÚc_fcé   ÚgeluÚc_proj)r!   r"   r	   ÚMultiheadAttentionÚattnrp   Úln_1Ú
Sequentialr   ÚLinearr~   ÚmlpÚln_2r‰   )r'   r‡   rˆ   r‰   r(   s       €r   r"   ÚResidualAttentionBlock.__init__°   s“   ø€ Ü‰ÑÔä×)Ò)¨'Ó:ˆŒ	Ü˜gÓ&ˆŒ	Ü—=’=Ü˜&¤"§)¢)¨G¸q±[Ó"AÐBÀVÌYË[ÐDYØ"¤B§I¢I¨g¸©k¸7Ó$CÐDðFó GóHˆŒô ˜gÓ&ˆŒ	Ø"�r   r,   c                 óÄ   • U R                   b.  U R                   R                  UR                  UR                  S9OS U l         U R	                  XUSU R                   S9S   $ )N)rt   ÚdeviceF)Úneed_weightsr‰   r   )r‰   Útort   r˜   r�   r‚   s     r   Ú	attentionÚ ResidualAttentionBlock.attention»   sR   € ØNRÏnÉnÑNh˜Ÿ™×*Ñ*°·±ÀÇÁÐ*ÑJÐnrˆŒØ�y‰y˜˜q¨uÀÇÁˆyÐOÐPQÑRÐRr   c                 óŽ   • XR                  U R                  U5      5      -   nXR                  U R                  U5      5      -   nU$ rs   )r›   r‘   r”   r•   r‚   s     r   r-   ÚResidualAttentionBlock.forward¿   s9   € Ø—‘˜tŸy™y¨›|Ó,Ñ,ˆØ—‘˜Ÿ™ 1›Ó&Ñ&ˆØˆr   )r�   r‰   r‘   r•   r”   rs   )r/   r0   r1   r2   Úintrv   r|   r"   r›   r-   r4   r5   r6   s   @r   r…   r…   ®   sM   ø† ñ	# ð 	#¨Sð 	#¸U¿\¹\÷ 	#ð 	#ðS˜5Ÿ<™<ô Sð˜Ÿ™÷ ò r   r…   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )ÚStyleAdapteréÅ   c           	      óÂ  >• [         TU ]  5         US-  n[        R                  " [	        U5       Vs/ s H  n[        X5      PM     sn6 U l        XPl        [        R                  " [        R                  " SXQ5      U-  5      U l        [        U5      U l        [        U5      U l        [        R                  " U[        R                  " X5      -  5      U l        g s  snf )Ng      à¿r   )r!   r"   r	   r’   r`   r…   Útransformer_layesÚ	num_tokenÚ	Parameterrv   ÚrandnÚstyle_embeddingrp   Úln_postÚln_preÚproj)	r'   ÚwidthÚcontext_dimÚnum_headÚn_layesr¥   ÚscaleÚ_r(   s	           €r   r"   ÚStyleAdapter.__init__Ç   s©   ø€ Ü‰ÑÔà˜‘ˆÜ!#§¢ÔbgÐhoÔbpÓ0qÒbpÐ]^Ô1GÈÖ1XÑbpÑ0qÐ!rˆÔØ"ŒÜ!Ÿ|š|¬E¯KªK¸¸9Ó,LÈuÑ,TÓUˆÔÜ  Ó'ˆŒÜ Ó&ˆŒÜ—L’L ¬¯ª°UÓ)HÑ!HÓIˆ�	ùò 1rs   ²Cc                 óè  • U R                   [        R                  " UR                  S   U R                  U R                   R                  S   4UR
                  S9-   n[        R                  " X/SS9nU R                  U5      nUR                  SSS5      nU R                  U5      nUR                  SSS5      nU R                  US S 2U R                  * S 2S S 24   5      nXR                  -  nU$ )Nr   éÿÿÿÿ)r˜   r   )Údimr   )r¨   rv   Úzerosr+   r¥   r˜   Úcatrª   Úpermuter¤   r©   r«   )r'   r,   r¨   s      r   r-   ÚStyleAdapter.forwardÒ   sÖ   € à×.Ñ.´·²Ø�W‰W�Q‰Z˜Ÿ™¨×)=Ñ)=×)CÑ)CÀBÑ)GÐHÐQR×QYÑQYñ2[ñ [ˆä�IŠI�qÐ*°Ñ2ˆØ�K‰K˜‹NˆØ�I‰I�a˜˜AÓˆØ×"Ñ" 1Ó%ˆØ�I‰I�a˜˜AÓˆà�L‰L˜š1˜tŸ~™~˜oÑ.²Ð1Ñ2Ó3ˆØ—	‘	‰Mˆàˆr   )r©   rª   r¥   r«   r¨   r¤   )i   i   rW   r   rŒ   rO   r6   s   @r   r¡   r¡   Å   s   ø† ÷	J÷ð r   r¡   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚResnetBlock_lightéâ   c                 óÎ   >• [         TU ]  5         [        R                  " XSSS5      U l        [        R
                  " 5       U l        [        R                  " XSSS5      U l        g )Nr   r   )r!   r"   r	   r   r=   r>   r?   r@   )r'   rD   r(   s     €r   r"   ÚResnetBlock_light.__init__ã   sG   ø€ Ü‰ÑÔÜ—i’i ¨A¨q°!Ó4ˆŒÜ—7’7“9ˆŒÜ—i’i ¨A¨q°!Ó4ˆ�r   c                 óp   • U R                  U5      nU R                  U5      nU R                  U5      nX!-   $ rs   )r=   r?   r@   rL   s      r   r-   ÚResnetBlock_light.forwardé   s0   € Ø�K‰K˜‹NˆØ�H‰H�Q‹KˆØ�K‰K˜‹Nˆà‰uˆr   )r?   r=   r@   rO   r6   s   @r   r»   r»   â   s   ø† õ5÷ð r   r»   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )Ú	extractoréñ   c                 ó¤  >• [         TU ]  5         [        R                  " XSSS5      U l        / U l        [        U5       H'  nU R
                  R                  [        U5      5        M)     [        R                  " U R
                  6 U l        [        R                  " X#SSS5      U l
        XPl        U R                  S:X  a  [        USS9U l        g g )Nr   r   TFr;   )r!   r"   r	   r   r<   r_   r`   rb   r»   r’   Úout_convrB   r   rC   )r'   rD   Úinter_crE   r^   rB   r±   r(   s          €r   r"   Úextractor.__init__ò   sŸ   ø€ Ü‰ÑÔÜ—y’y °°1°aÓ8ˆŒØˆŒ	Ü�w–ˆAØ�I‰I×ÑÔ.¨wÓ7Ö8ñ  ä—M’M 4§9¡9Ð-ˆŒ	ÜŸ	š	 '°!°Q¸Ó:ˆŒØŒ	Ø�9‰9˜ÓÜ& t°eÑ<ˆD�Mð r   c                 ó®   • U R                   S:X  a  U R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU$ rK   )rB   rC   r<   r_   rÅ   r‚   s     r   r-   Úextractor.forwardþ   sI   € Ø�9‰9˜ÓØ—‘˜aÓ ˆAØ�L‰L˜‹OˆØ�I‰I�a‹LˆØ�M‰M˜!Óˆàˆr   )r_   rB   rC   r<   rÅ   )FrO   r6   s   @r   rÂ   rÂ   ñ   s   ø† ÷
=÷ð r   rÂ   c                   ó<   ^ • \ rS rSr/ SQSS4U 4S jjrS rSrU =r$ )ÚAdapter_lighti  rS   r   rU   c                 óD  >• [         [        U ]  5         SU l        [        R
                  " U R                  5      U l        X0R                  U R                  -  -  U l        Xl        X l	        / U l
        SU l        [        [        U5      5       Hj  nUS:X  a/  U R                  R                  [        X1U   S-  X   USS95        M8  U R                  R                  [        XS-
     X   S-  X   USS95        Ml     [        R                   " U R                  5      U l
        g )NrW   Fr   rŒ   )rD   rÆ   rE   r^   rB   r   T)r!   rË   r"   rY   r	   r\   r]   r[   r#   r^   r_   rZ   r`   ra   rb   rÂ   rc   )r'   r#   r^   re   rh   r(   s        €r   r"   ÚAdapter_light.__init__	  sû   ø€ ÜŒm˜TÑ+Ô-Ø !ˆÔÜ×*Ò*¨4×+@Ñ+@ÓAˆŒØ!×&;Ñ&;¸d×>SÑ>SÑ&SÑTˆÔØ ŒØŒØˆŒ	ØˆŒä”s˜8“}Ö%ˆAØ�A‹vØ—	‘	× Ñ ¤°Àa¹[È!¹^ÐS[ÑS^ÐhoÐv{Ñ!|Ö}à—	‘	× Ñ ¤°¸1¹±ÀxÁ{ÐTUÁ~Ð]eÑ]hÐryð  AEñ  "Fö  Gñ	 &ô
 —M’M $§)¡)Ó,ˆ�	r   c                 ó  • U R                  U5      n/ n[        [        U R                  5      5       HK  nU R                  U   " U5      nUR                  S 5        UR                  S 5        UR                  U5        MM     U$ rs   )r]   r`   ra   r#   r_   rb   )r'   r,   rl   rh   s       r   r-   ÚAdapter_light.forward  si   € à�N‰N˜1ÓˆàˆÜ”s˜4Ÿ=™=Ó)Ö*ˆAØ—	‘	˜!’˜Q“ˆAØ�O‰O˜DÔ!Ø�O‰O˜DÔ!Ø�O‰O˜AÖñ	 +ð ˆr   )r_   r#   r[   r^   r]   rY   rZ   rO   r6   s   @r   rË   rË     s   ø† Ú 6ÀÀr÷ -÷"ð r   rË   )rv   Útorch.nnr	   Úcollectionsr   r   r   ÚModuler   r8   rQ   rp   r~   r…   r¡   r»   rÂ   rË   r{   r   r   Ú<module>rÓ      s»   ðã Ý Ý #ò
8ò
8ô�—‘ô ôD!�"—)‘)ô !ôH5ˆb�i‰iô 5ôp#�—‘ô #ô,�—	‘	ô ,ô˜RŸY™Yô ô.�2—9‘9ô ô:˜Ÿ	™	ô ô�—	‘	ô ô.�B—I‘Iõ r   