
    i&                         d Z ddgZddlZddlmZ ddlZddlZddlZddl	Z
ddlmZ ddlmZmZmZ ddlmZ d	d
lmZmZ ej,                  rddlmZ ddlmZ ddlmZ  G d dej:                        Z G d dee      ZdefdZ  G d de      Z!y)z&Processor used for ``online`` engines.OnlineProcessorOnlineParams    N)default_timer)gen_useragent) SearxEngineAccessDeniedExceptionSearxEngineCaptchaException#SearxEngineTooManyRequestsException)count_error   )EngineProcessorRequestParams)SearchQuery)ResultContainer)EngineResultsc                   P   e Zd ZU dZej
                  d   ed<   	 eeef   ed<   	 eeee	z  eeee	z  f   z  f   ed<   	 eeej                  f   ed<   	 eed<   	 edz  ed	<   	 eeef   ed
<   	 eed<   	 e	ed<   	 e	ed<   	 dej
                  d   z  ez  ed<   	 edz  ed<   	 eed<   y)
HTTPParamszHTTP request parameters)GETPOSTmethodheadersdatajsoncontentNurlcookiesallow_redirectsmax_redirectssoft_max_redirectsFverifyauthraise_for_httperror)__name__
__module____qualname____doc__tLiteral__annotations__dictstrintAnybytesbool     //root/searxng/searx/search/processors/online.pyr   r      s    !IIm$$#s(^"
sC#IS#)^ 444
55 sAEEz
 N 
tO#s(^(S199U##c)) *9r0   r   c                       e Zd ZdZy)r   z*Request parameters of a ``online`` engine.N)r"   r#   r$   r%   r/   r0   r1   r   r   Z   s    4r0   returnc                  "    di i i ddi dddddddS )	z2Default request parameters for ``online`` engines.r   r0    Fr   NT)r   r   r   r   r   r   r   r   r   r   r    r   r!   r/   r/   r0   r1   default_request_paramsr6   ^   s5      # r0   c            
            e Zd ZU dZdZeed<   def fdZde	de	fdZ
d	d
dededz  f fdZdefdZdededdfdZdededdde	de	f
dZ xZS )r   z'Processor class for ``online`` engines.onlineengine_typer3   c                 ~    | j                  t               | j                  j                         t        |          S )zThis method is called in a thread, and before the base method is
        called, the network must be set up for the ``online`` engines.)
start_timetimeout_limit)init_network_in_threadr   enginetimeoutsuperinit_engine)self	__class__s    r1   rA   zOnlineProcessor.init_enginev   s2     	##}dkkNaNa#bw"$$r0   r;   r<   c                     t         j                  j                  ||       t         j                  j                          t         j                  j	                  | j
                  j                         y )N)r;   )searxnetworkset_timeout_for_threadreset_time_for_threadset_context_network_namer>   name)rB   r;   r<   s      r1   r=   z&OnlineProcessor.init_network_in_thread|   sE    ,,]z,R++-..t{{/?/?@r0   search_queryr   engine_categoryNc                    t         |   ||      }||S i t               |}|d   }d|d<   d|d<   d|d<   d|d	<   t               |d
<   d|d<   | j                  j
                  rJ|j                  r>|j                  j                  }|j                  j                  xs |}| d| d| d|d<   | j                  j                  d|j                  dd             |S )zReturns a dictionary with the :ref:`request params <engine request
        online>` (:py:obj:`OnlineParams`), if the search condition is not
        supported by the engine, ``None`` is returned.r   zgzip, deflatezAccept-Encodingzno-cachezCache-Control1DNTz
keep-alive
Connectionz
User-Agentzen,en-US;q=0.7,en;q=0.3zAccept-Language,-z;q=0.7,en;q=0.3zHTTP Accept-Language: %sr5   )r@   
get_paramsr6   r   r>   send_accept_language_headerlocalelanguage	territoryloggerdebugget)	rB   rK   rL   base_paramsparamsr   _l_trC   s	           r1   rS   zOnlineProcessor.get_params   s   
 -2G,>|_,]J"8":JkJ#%4!"#-  , !.
 &?!";;22|7J7J$$--B$$..4"B,.4qAbT)IG%&4gkkBSUW6XYr0   r\   c                    |d   |d   |d   d}|j                  d      }|||d<   |j                  d      }|r||d<   d|v r|d   |d<   |j                  d|xs d	      }|j                  d
d      |d
<   |d   dk(  rt        j                  j                   }nAt        j                  j                  }|d   r|d   |d<   |d   r|d   |d<   |d   r|d   |d<    ||d   fi |}t	        |j
                        |kD  rt        |j                  xs d      }|j                  xs d}	|j                  j                  }
t        | j                  j                  dj                  t	        |j
                        |      ||	|
fd       |S )Nr   r   r    )r   r   r    r   r   r   r   r   r!   Tr   r   r   r   r   r   r5   z{} redirects, maximum: {})	secondary)rZ   rE   rF   postlenhistoryr*   status_codereason_phraser   hostr
   r>   rJ   format)rB   r\   request_argsr   r   r   reqresponserd   reasonhostnames              r1   _send_http_requestz"OnlineProcessor._send_http_request   s    i(i(6N*
 H%%+L" 

?3,9L) &.45F.GL*+ #)**-A=CUTU"V /5jj9NPT.U*+ (u$--##C--$$Cf~'-f~V$f~'-f~V$i *0*;Y' ve}55 x #55 h228b9K++1rF||((H  +223x7G7G3HJ\]fh/	 r0   queryzEngineResults|Nonec                     | j                   j                  ||       |d   sy | j                  |      }||_        | j                   j	                  |      S )Nr   )r>   requestrm   search_paramsrj   )rB   rn   r\   rj   s       r1   _search_basiczOnlineProcessor._search_basic   sU     	E6* e} **62 "({{##H--r0   result_containerr   c           	         | j                  ||       	 | j                  ||      }| j                  |||       y # t        j                  $ r}| j                  ||d       | j                  j                  dj                  |t        j                  j                  | j                  j                        j                               Y d }~y d }~wt        j                   t"        j$                  f$ ri}| j                  ||d       | j                  j                  dj                  t'               |z
  ||j(                  j*                               Y d }~y d }~wt        j,                  t        j.                  f$ rU}| j                  ||d       | j                  j1                  dj                  t'               |z
  ||             Y d }~y d }~wt2        t4        t6        f$ rC}| j                  ||d       | j                  j1                  |j8                         Y d }~y d }~wt:        $ rF}| j                  ||       | j                  j1                  dj                  |             Y d }~y d }~ww xY w)NT)suspendzSSLError {}, verify={}zEHTTP requests timeout (search duration : {0} s, timeout: {1} s) : {2}zBrequests exception (search duration : {0} s, timeout: {1} s) : {2}zexception : {0})r=   rr   extend_containersslSSLErrorhandle_exceptionrX   errorrg   rE   rF   get_networkr>   rJ   r   httpxTimeoutExceptionasyncioTimeoutErrorr   rC   r"   	HTTPErrorStreamError	exceptionr   r	   r   message	Exception)rB   rn   r\   rs   r;   r<   search_resultses           r1   searchzOnlineProcessor.search   s     	##J>!	?!//v>N!!"2JO|| 	v!!"2At!DKK6==aAZAZ[_[f[f[k[kAlAsAstuu&&(<(<= 	!!"2At!DKKW^^!Oj0-AUAU 
 !2!23 	!!"2At!DKK!!T[[!Oj0-  (/,
 	-
 !!"2At!DKK!!!)),, 	?!!"2A6KK!!"3":":1"=>>	?sH   %: I3A:C&I32AE&I3<AGI3#9H!!I3-<I..I3)r"   r#   r$   r%   r9   r*   r(   r.   rA   floatr=   r   rS   rm   rr   r   __classcell__)rC   s   @r1   r   r   q   s    1K%T %A Au A} s |^bOb @9 9v.3 . .AU . +?+? +? ,	+?
 +? +?r0   )"r%   __all__typingr&   timeitr   r~   rw   r|   searx.networkrE   searx.utilsr   searx.exceptionsr   r   r	   searx.metrics.error_recorderr
   abstractr   r   TYPE_CHECKINGsearx.search.modelsr   searx.resultsr   searx.result_typesr   	TypedDictr   r   r6   r   r/   r0   r1   <module>r      s    ,n
-     
   % 
 5 4??/-0: :z5:} 5
 &i?o i?r0   