
    idE                        U d gZ ddlZddlmZ ddlZddlZddlZddlm	Z	 ddl
Z
ddlmZmZ ddlmZ ddlmZmZmZ dd	lmZ  ej*                  d
      ZdZi Zeedf   ed<   ddddddddddd
ZdddZej:                   G d d             Zd dedz  ddfdZd Z 	 	 d!de!eeejD                  f      deeejD                  f   ddfdZ#ejH                  d        Z% e       ee<   y)"get_network    N)	Generator)cycle)logger
sxng_debug)SXNG_Response   )
new_clientget_loopAsyncHTTPTransportNoHttp)raise_for_httperrornetwork__DEFAULT__NetworkNETWORKSzhttp://zhttps://z	socks4://z	socks5://z
socks5h://)
httphttpssocks4socks5socks5hzhttp:zhttps:zsocks4:zsocks5:zsocks5h:0.0.0.0::)ipv4ipv6c                      e Zd ZdZi Z	 	 	 	 	 	 	 	 	 	 	 	 	 d+dedededededed	ee	eef   z  dz  d
edee
e   z  dz  dedededefdZd Zdee   fdZd Zdeeee
e   f      fdZdeeeeef   df   eef   fdZdej(                  fdZedej.                  defd       Zd,dedz  dedz  dej.                  fdZd Zede	eej8                  f   de	eej8                  f   fd       Zede	eej8                  f   fd        Zdej(                  d!edefd"Z dej(                  fd#Z!d$ed%ed&edej8                  def
d'Z"d%ed&edej8                  defd(Z#d%ed&efd)Z$e%d*        Z&y)-r   )enable_httpverifyenable_http2max_connectionsmax_keepalive_connectionskeepalive_expirylocal_addressesproxiesusing_tor_proxymax_redirectsretriesretry_on_http_error_local_addresses_cycle_proxies_cycle_clients_loggerNr   r   r   r   r    r!   r#   r$   r"   r&   r'   r%   logger_namec                 r   || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        || _
        || _        | j                         | _        | j                         | _        i | _        |rt#        j$                  |      nt"        | _        | j)                          y N)r   r   r   r   r    r!   r#   r$   r"   r&   r'   r%   get_ipaddress_cycler(   get_proxy_cyclesr)   r*   r   getChildr+   check_parameters)selfr   r   r   r   r    r!   r#   r$   r"   r&   r'   r%   r,   s                 &/root/searxng/searx/network/network.py__init__zNetwork.__init__D   s    $ '(.)B& 0..#6 *&*&>&>&@#"3357Bv{3    c                     | j                         D ]2  }d|v rt        j                  |d       t        j                  |       4 | j                  ,t        | j                  t        t        f      st        d      y y )N/Fz(proxies type has to be str, dict or None)	iter_ipaddresses	ipaddress
ip_network
ip_addressr#   
isinstancestrdict
ValueError)r3   addresss     r4   r2   zNetwork.check_parametersh   sq    ,,. 	.Gg~$$We4$$W-		. <<#Jt||c4[,QGHH -R#r6   returnc              #   h   K   | j                   }|sy t        |t              r|g}|E d {    y 7 wr.   )r"   r=   r>   )r3   r"   s     r4   r9   zNetwork.iter_ipaddressesr   s2     ..os+./O"""s   (202c              #     K   	 d}| j                         D ]i  }d|v r<t        j                  |d      j                         D ]  }t	        |       |dz  } Ct        j
                  |      }t	        |       |dz  }k |dk(  rd  w)Nr   r8   Fr	   )r9   r:   r;   hostsr>   r<   )r3   countrA   as       r4   r/   zNetwork.get_ipaddress_cyclez   s     E002 '>&11'5AGGI #!!f
# ",,W5Aa&LQJE z
 s   B
Bc              #      K   | j                   sy t        | j                   t              rd| j                   gf y | j                   j                         D ]4  \  }}t        j                  ||      }t        |t              r|g}||f 6 y w)Nzall://)r#   r=   r>   itemsPROXY_PATTERN_MAPPINGget)r3   pattern	proxy_urls      r4   iter_proxieszNetwork.iter_proxies   s~     ||dllC(T\\N**&*ll&8&8&: )"488'Ji-!*Iy((	)s   BB.c              #      K   i }| j                         D ]  \  }}t        |      ||<    	 t        d |j                         D               #w)Nc              3   <   K   | ]  \  }}|t        |      f  y wr.   )next).0rL   proxy_url_cycles      r4   	<genexpr>z+Network.get_proxy_cycles.<locals>.<genexpr>   s     p=UWo$"78ps   )rN   r   tuplerI   )r3   proxy_settingsrL   
proxy_urlss       r4   r0   zNetwork.get_proxy_cycles   sZ     +-#'#4#4#6 	8GZ&+J&7N7#	8pYgYmYmYoppp s   AAresponsec           
      <  K   |j                   }|j                   d|j                   }|j                   d| }|j                  j                  d      }|rd| dnd}| j                  j                  d|j                   d|j                   d| d|        y w)	N zContent-Typez () zHTTP Request: z "")
requeststatus_codereason_phrasehttp_versionheadersrK   r+   debugmethodurl)r3   rX   r^   statusresponse_linecontent_types         r4   log_responsezNetwork.log_response   s     ""(()8+A+A*BC#0016(;''++N;/;L>+^GNN+;1W[[MM?Z[\h[ijks   BBclientc                   K   |t         j                  v rt         j                  |   S d}| j                  j                         D ]8  }t	        |t
              rt        |d      rt        |j                  dd      r8 y | j                  dd       d {   }|j                         d   sd}|t         j                  |<   |S 7 .w)	NT_pool_rdnsFz#https://check.torproject.org/api/ip<   )timeoutIsTor)
r   _TOR_CHECK_RESULT_mountsvaluesr=   r   getattrrl   rK   json)rj   r#   result	transportrX   s        r4   check_tor_proxyzNetwork.check_tor_proxy   s     g///,,W55..0 
	I)%=>y'*w	0 
	  $ISUVV}}w'F-3!!'*	 Ws   BCC/Cc                   K   || j                   n|}|| j                  n|}t        | j                        }t        | j                        }||||f}t
        r| j                  nd }|| j                  vs| j                  |   j                  rt        | j                  || j                  | j                  | j                  | j                  t        |      |d||      }| j                   rG| j#                  ||       d {   s-|j%                          d {    t'        j(                  d      || j                  |<   | j                  |   S 7 O7 9w)Nr   z,Network configuration problem: not using Tor)r   r%   rQ   r(   r)   r   ri   r*   	is_closedr
   r   r   r   r    r!   r?   r$   rx   aclosehttpx
ProxyError)r3   r   r%   local_addressr#   keyhook_log_responserj   s           r4   
get_clientzNetwork.get_client   s7     &F.;.C**T889t**+}mW=1;D--dmm#t}}S'9'C'C  !!$$..%%W!F ##$2F2Fvw2W,W,Wmmo%%&&'UVV!'DMM#}}S!!	 -X%s$   C=E?E EE8EEc                    K   d }t        j                  | j                  j                         D cg c]
  } ||       c}ddi d {    y c c}w 7 
w)Nc                 t   K   	 | j                          d {    y 7 # t        j                  $ r Y y w xY wwr.   )r{   r|   	HTTPError)rj   s    r4   close_clientz$Network.aclose.<locals>.close_client   s/     mmo%%?? s(   8  8 5858return_exceptionsF)asynciogatherr*   rs   )r3   r   rj   s      r4   r{   zNetwork.aclose   sH     	 nn$--BVBVBXY|F3YsmrsssYss   /AA 
A
AAkwargsc                     i }d| v r| j                  d      |d<   d| v r| j                  d      |d<   d| v r| j                  d      | d<   |S )Nr   r%   allow_redirectsfollow_redirects)pop)r   kwargs_clientss     r4   extract_kwargs_clientszNetwork.extract_kwargs_clients   s`    +-v'-zz(';N8$f$.4jj.IN?+&)/4E)FF%&r6   c                 "    d}d| v r| d   }| d= |S )NTr    )r   do_raise_for_httperrors     r4   extract_do_raise_for_httperrorz&Network.extract_do_raise_for_httperror   s+    !% F*%+,A%B",-%%r6   r   c                 P   t        |t        j                        r<t        j                  t
        |      }|j                   |_        |r	 t        |       |S |S #  | j                  j                  d|j                  j                   d|j                  j                           xY w)NzHTTP Request failed: rZ   )r=   r|   Responsetcastr   is_errorokr   r+   warningr^   rd   re   )r3   rX   r   s      r4   patch_responsezNetwork.patch_response   s    h/vvmX6H '///HK &'1 xLL((+@AQAQAXAX@YYZ[c[k[k[o[oZp)qrs   
A AB%c                    | j                   du rd|j                  cxk  rdk  sgn t        | j                   t              r|j                  | j                   v s3t        | j                   t              r|j                  | j                   k(  ryy)NTi  iW  F)r'   r_   r=   listint)r3   rX   s     r4   is_valid_responsezNetwork.is_valid_response  ss     %%-#9M9M2TQT2T433T:x?S?SW[WoWo?o433S9h>R>RVZVnVn>nr6   streamrd   re   c                   K   | j                   }d}t        j                  |      }t        j                  |      }|dk\  r | j                  di | d {   }	|j                  dd       }
t        j                  |
      |	_        	 |r |	j                  ||fi |S  |	j                  ||fi | d {   }| j                  |      s|dk  r| j                  ||      S 	 |dz  }|dk\  ry y 7 7 <# t        j                  $ rO}|s<d}|	j                          d {  7   | j                  j!                  d       Y d }~	|dk  r|Y d }~nd }~wt        j"                  t        j$                  f$ r}|dk  r|Y d }~d }~ww xY ww)NFr   cookiesTz@httpx.RemoteProtocolError: the server has disconnected, retryingr	   r   )r&   r   r   r   r   r   r|   Cookiesr   r   r^   r   r   RemoteProtocolErrorr{   r+   r   RequestErrorHTTPStatusError)r3   r   rd   re   r   r&   was_disconnectedr   r   rj   r   rX   es                r4   call_clientzNetwork.call_client  s|    ,, !(!G!G!O 77?l*4??<^<<FjjD1G"]]73FN(6==???!/!Fv!FF))(3w!|..x9OPP 8D qLG1 l< G ,, 	' (,$ --/))LL(()kla<G  &&(=(=> a<G  s   AF
C/0F
C3 F
C3 4C15+C3  F
-F
1C3 3FED  E>F
EF
&F6F=F
FF
c                 H   K    | j                   d||fi | d {   S 7 w)NFr   r3   rd   re   r   s       r4   r^   zNetwork.request/  s(     %T%%eVSCFCCCC   " "c                 H   K    | j                   d||fi | d {   S 7 w)NTr   r   s       r4   r   zNetwork.stream2  s(     %T%%dFCB6BBBBr   c                    K   t        j                  t        j                         D cg c]  }|j	                          c}ddi d {    y c c}w 7 
w)Nr   F)r   r   r   rs   r{   )clsr   s     r4   
aclose_allzNetwork.aclose_all5  s=     nnx?PQGw~~/QkejkkkQks   &AA
A	A
A)TTFNNNNFNr   F   NNN)'__name__
__module____qualname__	__slots__rq   boolr   floatr>   r?   r   r5   r2   r   r9   r/   rU   rN   r0   r|   r   ri   staticmethodAsyncClientrx   r   r{   r   Anyr   r   r   r   r   r   r^   r   classmethodr   r   r6   r4   r   r   ,   s   I& 
 !"#)-"&/3 %26$)"  "  	" 
 "  "  $'"   "  tCH~%,"  "  tCy4/"  "  ""  "  " HI#)C. #)ic49n(=> )q)E%S/32F,Gc,Q"R ql5>> l e&7&7 T  ."td{ "#PT* "`e`q`q "6t 	tCJ'7 	Daee<L 	 	 &tCJ/? & &u~~ t Xe  %..  c  quu Yf >DC Dc DQUU D} DC3 CS C l lr6   namerB   c                 <    t         j                  | xs t              S r.   )r   rK   DEFAULT_NAME)r   s    r4   r   r   :  s    <<,--r6   c                      d } t        j                   |        t                     }|j                         }|dkD  rt	        d      y )Nc                     K   d} t         j                         D ](  }|j                  s	 |j                          d {    * | S 7 # t        $ r# |j
                  j                  d       | dz  } Y Zw xY ww)Nr   Errorr	   )r   rs   r$   r   	Exceptionr+   	exception)exception_countr   s     r4   checkz*check_network_configuration.<locals>.check?  sx     ( 	)G&&)!,,...	) 	 /  )OO--g6#q(O)s8   &A7AAAA7A)A41A73A44A7r   zInvalid network configuration)r   run_coroutine_threadsafer   rv   RuntimeError)r   futurer   s      r4   check_network_configurationr   >  sC    	 --egxzBFmmoO:;; r6   settings_enginessettings_outgoingc                     ddl m ddlm}  xs |d    |xs |d   }d|d   |d   |d	   |d
   |d   |d   |d   |d   |d   |d   ddddt        t
        t        j                  f   dt
        d z  ffd} fd}t        r
t                t        j                           |i d      t        t        <    |ddid      t        d<    |ddid      t        d<   |d   j                         D ]  \  }} |||      t        |<     |       D ]w  \  }}}|Mi }j                         D ]&  \  }	}
t        ||	      rt        ||	      ||	<   "|
||	<   (  |||      t        |<   Vt        |t              sg |||      t        |<   y  |       D ]'  \  }}}t        |t
              st        |   t        |<   ) dt        vr'j!                         }d|d<    ||d      t        d<   y y ) Nr   )engines)settingsr   outgoingFr   r   pool_connectionspool_maxsizer!   
source_ipsr$   r#   r%   r&   )r   r   r   r   r    r!   r"   r$   r#   r%   r&   r'   paramsr,   c                 p    i }|j                         |j                  |        |r||d<   t        di |S )Nr,   r   )updater   )r   r,   rv   default_paramss      r4   new_networkzinitialize.<locals>.new_networkn  s;    n%f$/F=!   r6   c               3   v   K   D ]/  } | d   }j                  |      }|t        |dd       }|||f 1 y w)Nr   r   )rK   rt   )engine_specengine_nameenginer   r   r   s       r4   iter_networksz!initialize.<locals>.iter_networksw  sQ     + 	/K%f-K[[-F~fi6Gvw..	/s   69default)r,   r"   r   r   r   r   networksimage_proxyr.   )searx.enginesr   searxr   r?   r>   r   r   r   doneclearr   rI   hasattrrt   r=   copy)r   r   r   r   r   network_namer   r   r   attribute_nameattribute_valueimage_proxy_paramsr   r   s   `           @@r4   
initializer   P  sF   
 & (>8I+>)AXj-A
 #H-).9,-?@%6~%F-.@A,\:,->?$Y/*?;$Y/$(N!Daee, !3: !/ NN(CH\"$5y#AvVHV"$5t#<&QHV "3:!>!D!D!F Pg!,W,!OP )6 
R$VW?G3A3G3G3I >/6>2.5fn.MGN+.=GN+	>
 %0[$QH[!&$/[$QH[!
R )6 6$VWgs#$,W$5H[!6 H$+002-2>*"-.@m"\ %r6   c                      	 t               } | r9t        j                  t        j	                         |       }|j                  d       t        j                          y# t        j                          w xY w)a  Close all HTTP client

    Avoid a warning at exit
    See https://github.com/encode/httpx/pull/2026

    Note: since Network.aclose has to be async, it is not possible to call this method on Network.__del__
    So Network.aclose is called here using atexit.register
       N)r   r   r   r   r   rv   r   r   )loopr   s     r4   r   r     sM    z55g6H6H6JDQFMM!s   AA A2r.   r   )&__all__typingr   collections.abcr   atexitr   r:   	itertoolsr   r|   r   r   r   searx.extended_typesr   rj   r
   r   r   r   r1   r   r   r?   r>   __annotations__rJ   ADDRESS_MAPPINGfinalr   r   r   r   r   r   registerr   r   r6   r4   <module>r      sI  
 =/  %      $ . B B 4 
	#!#$sI~
 #   %d3 Jl Jl 	JlZ.cDj .I .<& 04*.T]4QUU
+,T]CJ'T] 
T]n  & ! r6   