
    iC                        U d Z ddlZddlZddlmZ ddlmZmZ ddlm	Z	 ddl
mZmZ ddlZddlZddlZddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZm Z m!Z!m"Z"m#Z#m$Z$ dddddddZ%dZ&	 ddgZ'dZ(dZ)	 dZ*dZ+dddddZ,ddddZ-dZ.e.dz   Z/d Z0	 ee1d!<   	 d" Z2d#Z3	 d$ Z4d% Z5d&e6d'e7e6edz  f   fd(Z8d) Z9d* Z:d'e;e6ejx                  f   dz  fd+Z=d, Z>d-efd.Z?y)/u^	  Startpage's language & region selectors are a mess ..

.. _startpage regions:

Startpage regions
=================

In the list of regions there are tags we need to map to common region tags::

  pt-BR_BR --> pt_BR
  zh-CN_CN --> zh_Hans_CN
  zh-TW_TW --> zh_Hant_TW
  zh-TW_HK --> zh_Hant_HK
  en-GB_GB --> en_GB

and there is at least one tag with a three letter language tag (ISO 639-2)::

  fil_PH --> fil_PH

The locale code ``no_NO`` from Startpage does not exists and is mapped to
``nb-NO``::

    babel.core.UnknownLocaleError: unknown locale 'no_NO'

For reference see languages-subtag at iana; ``no`` is the macrolanguage [1]_ and
W3C recommends subtag over macrolanguage [2]_.

.. [1] `iana: language-subtag-registry
   <https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry>`_ ::

      type: language
      Subtag: nb
      Description: Norwegian Bokmål
      Added: 2005-10-16
      Suppress-Script: Latn
      Macrolanguage: no

.. [2]
   Use macrolanguages with care.  Some language subtags have a Scope field set to
   macrolanguage, i.e. this primary language subtag encompasses a number of more
   specific primary language subtags in the registry.  ...  As we recommended for
   the collection subtags mentioned above, in most cases you should try to use
   the more specific subtags ... `W3: The primary language subtag
   <https://www.w3.org/International/questions/qa-choosing-language-tags#langsubtag>`_

.. _startpage languages:

Startpage languages
===================

HTTP ``Accept-Language`` header (``send_accept_language_header``):
  The displayed name in Startpage's settings page depend on the location of the
  IP when ``Accept-Language`` HTTP header is unset.

  Startpage tries to guess user's language and territory from the HTTP
  ``Accept-Language``.  Optional the user can select a search-language (can be
  different to the UI language) and a region filter.

  In :py:obj:`fetch_traits` we use::

    'Accept-Language': "en-US,en;q=0.5",
    ..

  to get uniform names independent from the IP).

.. _startpage categories:

Startpage categories
====================

Startpage's category (for Web-search, News, Videos, ..) is set by
:py:obj:`startpage_categ` in  settings.yml::

  - name: startpage
    engine: startpage
    startpage_categ: web
    ...

.. hint::

  Supported categories are ``web``, ``news`` and ``images``.

    N)OrderedDict)datetime	timedelta)loads)	combining	normalize)EngineCache)EngineTraits)SearxEngineCaptchaException)
region_tag)get)
eval_xpathextrextract_textgen_useragenthtml_to_texthumanize_bytesremove_pua_from_strzhttps://startpage.comQ2333295FHTML)websitewikidata_idofficial_api_documentationuse_official_apirequire_api_keyresultswebgeneralT   dwmy)dayweekmonthyear10)r         zhttps://www.startpage.comz
/sp/searchz//form[@id="search"]CACHEc                     t        d      ay )N	startpage)r	   r,   )_s    (/root/searxng/searx/engines/startpage.pyinitr1      s    
 $E    i  c                    t         j                  d      }|rt        j                  d|       |S t        dz   }t        j                  d|       i | d   }t        j                  d|       t        ||      }t        |j                        j                  d      rt        d	
      t        j                  j                  |j                        }	 t        |t        dz         d   }t        |      }t        j                  d|       t         j!                  d|t"               |S # t        $ r4}t        j                  d       t        d|j                  z  
      |d}~ww xY w)a"  Get an actual ``sc`` argument from Startpage's search form (HTML page).

    Startpage puts a ``sc`` argument on every HTML :py:obj:`search form
    <search_form_xpath>`.  Without this argument Startpage considers the request
    is from a bot.  We do not know what is encoded in the value of the ``sc``
    argument, but it seems to be a kind of a *timestamp*.

    Startpage's search form generates a new sc-code on each request.  This
    function scrapes a new sc-code from Startpage's home page every
    :py:obj:`sc_code_cache_sec` seconds.SC_CODEz#get_sc_code: using cached value: %s/z+get_sc_code: querying new sc timestamp @ %sheadersz get_sc_code: request headers: %s)r6   $https://www.startpage.com/sp/captchazCget_sc_code: got redirected to https://www.startpage.com/sp/captcha)messagez//input[@name="sc"]/@valuer   zEsuspend startpage API --> https://github.com/searxng/searxng/pull/695z<get_sc_code: [PR-695] querying new sc timestamp failed! (%s)Nzget_sc_code: new value is: %s)keyvalueexpire)r,   r   loggerdebugbase_urlstrurl
startswithr   lxmlhtml
fromstringtextr   search_form_xpath
IndexErrorsetsc_code_cache_sec)paramssc_code
get_sc_urlr6   respdomexcs          r0   get_sc_coderP      s<    ii	"G:GDCJ
LL>
K#	"#G
LL3W=z7+D 488} FG)Y
 	
 ))

tyy
)CS"36R"RSTUV 'lG
LL0':	II)73DIEN  \])RUYU]U]]
	s   D( (	E%1/E  E%c           	         t         j                  |d   d      }t         j                  |d   d      }t        |d   d<   t        dz   |d   d<   | t        dt        |      t        j                  |d	   d
      dddd}|r
||d<   ||d<   |d   dkD  r|d   |d<   d|d<   d}t               }d|d<   t        |d      |d<   d|d<   d|d<   d|d<   d|d<   d|d<   d|z  |d<   d |d!<   d|d"<   d#|d$<   |r
||d<   ||d%<   |r||d&<   d'j                  |j                         D cg c]  }d(|z  	 c}      |d)   d*<   t        j                  d+|d)   d*          t        j                  d,|       ||d-<   d.|d/<   t        |d0<   |S c c}w )1a  Assemble a Startpage request.

    To avoid CAPTCHAs we need to send a well formed HTTP POST request with a
    cookie. We need to form a request that is identical to the request built by
    Startpage's search form:

    - in the cookie the **region** is selected
    - in the HTTP POST data the **language** is selected

    Additionally the arguments form Startpage's search form needs to be set in
    HTML POST data / compare ``<input>`` elements: :py:obj:`search_form_xpath`.
    searxng_localezen-USenr6   Originr5   Refererdevice
time_range r(   )querycattsc	with_dateabpabdabelanguageluipagenor*   pagezstartpage.udogsegmentworld	date_time
safesearchdisable_family_filterr)   disable_open_in_new_windowenable_post_methodenable_proxy_safety_suggestenable_stay_controlinstant_answerszs/device/%s/lang_homepage10num_of_resultssuggestionscelsiuswt_unitlanguage_uisearch_results_regionN1Nz%sEEE%scookiespreferenceszcookie preferences: %szdata: %sdataPOSTmethodr@   )traits
get_regionget_languager>   startpage_categrP   time_range_dictr   r   safesearch_dictjoinitemsr<   r=   
search_url)rY   rJ   engine_regionengine_languageargsro   cookiexs           r0   requestr      s    %%f-=&>HM))&1A*BDIO"*F9h#+c>F9i  &!$(()=rB	D *Z%Uh!h'V*Y M]F!F;&5f\6J&KF"#+.F'(#&F ,/F()$'F ! #F,}<F?#FF=!F9,z /}*7&'',zz&,,.2YQ9q=2Y'ZF9m$
LL)6)+<]+KL
LLT"F6NF8F5MM 3Zs   E>contentreturnc           	         d }t        j                  d|       rG| j                  d      dz   }| d|dz
   }| |d  } 	 t        j                  j                  |d      }| |fS t        j                  d|       rm| j                  d      dz   }| d|dz
   }t        j                         t        t        t        j                  d	|      j                               
      z
  }| |d  } | |fS # t        $ r Y | |fS w xY w)Nz9^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]{2} [0-9]{4} \.\.\. z...   r      T)dayfirstz^[0-9]+ days? ago \.\.\. z\d+)days)rematchfinddateutilparserparse
ValueErrorr   nowr   intgroup)r   published_datedate_posdate_strings       r0   _parse_published_dater   ,  s   N 
xxLgV<<&*a(Q,/()$	%__22;2NN N"" 
.	8<<&*a(Q,/ ")RXXfk=Z=`=`=b9c*dd ()$N""  	 N""	s   !C' '	C65C6c                 ~    t        | j                  d            }t        |      \  }}| d   t        | d         ||dS )NdescriptionclickUrltitle)r@   r   r   publishedDate)r   r   r   )resultr   r   s      r0   _get_web_resultr   I  sJ    6::m45G27;G] j!fWo.&	 r2   c                 &   t        t        | d               }t        t        | j                  d                  }d }| j                  d      rt        j                  | d   dz        }d }| j                  d      rt
        | d   z   }| d   ||||dS )Nr   r   datei  thumbnailUrlr   )r@   r   r   r   	thumbnail)r   r   r   r   fromtimestampr>   )r   r   r   r   r   s        r0   _get_news_resultr   U  s    VG_ =>E!,vzz-/H"IJGMzz& ..vf~/DELzz.!&"88 j!&! r2   c           
         | j                  d      }|sy d }| j                  d      rt        | d   z   }d }| j                  d      r| j                  d      r| d    d| d    }d }| j                  d      r@dj                  t        t        j
                  | d               }t        t        |            }d|t        | d	         d| j                  d
      ||| j                  d      |d	S )NaltClickUrlr   widthheightr   filesizerX   zimages.htmlr   rawImageUrlformat)	templater@   r   r   img_srcthumbnail_src
resolution
img_formatr   )	r   r>   r   filterr?   isdigitr   r   r   )r   r@   r   r   r   size_strs         r0   _get_image_resultr   k  s    
**]
#CLzz.!&"88Jzz'vzz(3w(&*:);<
Hzz*776#++vj/ABC!#h-0 "fWo.::m,% jj*
 
r2   c                    t         j                         }dt        | j                  d| dd      z   dz   }| j                  j                  dd      j                  d      r
t               t        |      }|j                  d	i       j                  d
i       j                  di       }g }|j                  dg       D ]  }|j                  dg       D ]o  }|d   dk(  r|j                  t        |             &|d   dk(  r|j                  t        |             Id|d   v sQt        |      }|s_|j                  |       q  |S )N{z'React.createElement(UIStartpage.AppSerpz, {z}})z}}LocationrX   r7   render	presenterregionsmainliner   display_typez
web-googlez	news-bingimages)r   
capitalizer   rE   r6   r   rA   r   r   appendr   r   r   )rM   categresults_rawresults_jsonresults_objr   results_categitems           r0   responser     s@   &&(ETYY*QRWQXX\(]_deehllK||
B'223YZ)++%L""8R044["EII)UWXKG$R8 	)!%%i4 	)D^,<t45~.+=/56]>::(.NN4(	)	) Nr2   engine_traitsc           
      ,   t               dd}t        d|d      }|j                  st        d      t        j
                  j                  |j                        }g }|j                  d      D ]"  }|j                  |j                  d             $ |D ]  }|d	k(  r	d
dij                  ||      }d|v rY|j                  d      \  }}	|	j                  d      d   }	t        t        j                  j                  |dz   |	z   d            }
n+	 t        t        j                  j                  |d            }
| j"                  j                  |
      }|r||k7  rt!        d|
d|d|       || j"                  |
<    t        j                  d      j$                  j'                         D ci c]  \  }}|j)                         | }}}t+        d t        j,                  j/                               D ]  }t        j                  |      j1                         }|st!        d| d       8|j)                         }|||<   dj3                  t+        d t5        d|                  }t7        |      t7        |j9                               k(  s|||<    |j;                  dddddd d!       d"h}|j                  d#      D ]  }|j                  d      }||v rt=        |      j)                         }|j                  |      }
|
|j                  |      }
|
Y| j$                  j                  |
      }|r||k7  rt!        d|
d|d|       || j$                  |
<    y$# t        j                  $ r t!        d|z         Y w xY wc c}}w )%zgFetch :ref:`languages <startpage languages>` and :ref:`regions <startpage
    regions>` from Startpage.zen-US,en;q=0.5)z
User-AgentzAccept-Languagez%https://www.startpage.com/do/settingsr   )r6   timeoutz"Response from Startpage is not OK.zF//form[@name="settings"]//select[@name="search_results_region"]/optionr:   allno_NOnb_NO-r/   )sepz=IGNORE: can't determine babel locale of startpage's locale %szCONFLICT: babel z --> z, rS   c                 *    | j                  d      dk(  S )Nr/   r   )r   )	lang_codes    r0   <lambda>zfetch_traits.<locals>.<lambda>  s    )..-3 r2   z.IGNORE: language name of startpage's language z is unknown by babelrX   c                     t        |        S )N)r   )cs    r0   r   zfetch_traits.<locals>.<lambda>  s    y|3C r2   NFKDpt_BRzh_Hantkomlnbsi)	brazilianfantizhengwenhangulmalayamnorsk	sinhalese
english_ukz9//form[@name="settings"]//select[@name="language"]/optionN)r   r   okRuntimeErrorrB   rC   rD   rE   xpathr   splitr   babelLocaler   UnknownLocaleErrorprintr   	languagesr   lowerr   
localedatalocale_identifiersget_language_namer   r   lenencodeupdater   )r   r6   rM   rN   sp_region_namesoptioneng_tagbabel_region_taglrsxng_tagconflictr   namecatalog_engine2codenative_nameunaccented_nameskip_eng_tagss                     r0   fetch_traitsr	    s    $o+G
 /D
 77?@@
))

tyy
)C O))de 4vzz'234 # 2e#W-11'7C""#))#.DAqR A!%,,"4"4QWq[c"4"JKH%ell&8&89Is&8&ST !((,,X67"8WUV*1h'/26 KP,,W[J\JfJfJlJlJnoy$4::<2oo 3++- =	 ll9-??AB9+Mabc!'')+4K( ''&)CYvWbEc"de3'='='?#@@3<0!=(  !&	
  	M ))WX 4**W%m#F#))+&**73*..t4H **..x87"8WUV,3)%4q ++ UX__` ps   *M(3N(!NN)@__doc__r   typingr[   collectionsr   r   r   jsonr   unicodedatar   r   babel.localedatar   dateutil.parserr   	lxml.htmlrB   searx.enginelibr	   searx.enginelib.traitsr
   searx.exceptionsr   searx.localesr   searx.networkr   searx.utilsr   r   r   r   r   r   r   aboutr   
categoriespagingmax_pagetime_range_supportrh   r   r   r>   r   rF   __annotations__r1   rI   rP   r   r?   tupler   r   r   dictAnyr   r   r	   r2   r0   <module>r"     sP  Rj 
  # (  ,    ' / 8 $    '"&	  
	 O 
sS#Fcc* '$

 +   %   L.bEP#3 #5ho1E+F #:	,c155j!1D!8 >2r4 r4r2   