
    i8>                     :   U d Z ddlZddlZddlmZmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ dddddddZdZg ZdZej<                  d   ed<   	 dZ e!ed<   	 dZ"	 dZ#	 dZ$	 dZ%ddddZ&dZ'	 dddddZ(e)e!e!f   ed <   d!e!d"e)e!ejT                  f   d#dfd$Z+d%e!dz  fd&Z,d'e!d#e)e!ejT                  f   fd(Z-d)ed#efd*Z.d)ed#efd+Z/d)ed#efd,Z0d-e)e!ejT                  f   d#efd.Z1d-e)e!ejT                  f   d#efd/Z2d0efd1Z3y)2a
  Brave supports the categories listed in :py:obj:`brave_category` (General,
news, videos, images).  The support of :py:obj:`paging` and :py:obj:`time range
<time_range_support>` is limited (see remarks).

Configured ``brave`` engines:

.. code:: yaml

  - name: brave
    engine: brave
    ...
    brave_category: search
    time_range_support: true
    paging: true

  - name: brave.images
    engine: brave
    ...
    brave_category: images

  - name: brave.videos
    engine: brave
    ...
    brave_category: videos

  - name: brave.news
    engine: brave
    ...
    brave_category: news

  - name: brave.goggles
    time_range_support: true
    paging: true
    ...
    brave_category: goggles


.. _brave regions:

Brave regions
=============

Brave uses two-digit tags for the regions like ``ca`` while SearXNG deals with
locales.  To get a mapping, all *officiat de-facto* languages of the Brave
region are mapped to regions in SearXNG (see :py:obj:`babel
<babel.languages.get_official_languages>`):

.. code:: python

    "regions": {
      ..
      "en-CA": "ca",
      "fr-CA": "ca",
      ..
     }


.. note::

   The language (aka region) support of Brave's index is limited to very basic
   languages.  The search results for languages like Chinese or Arabic are of
   low quality.


.. _brave googles:

Brave Goggles
=============

.. _list of Goggles: https://search.brave.com/goggles/discover
.. _Goggles Whitepaper: https://brave.com/static-assets/files/goggles.pdf
.. _Goggles Quickstart: https://github.com/brave/goggles-quickstart

Goggles allow you to choose, alter, or extend the ranking of Brave Search
results (`Goggles Whitepaper`_).  Goggles are openly developed by the community
of Brave Search users.

Select from the `list of Goggles`_ people have published, or create your own
(`Goggles Quickstart`_).


.. _brave languages:

Brave languages
===============

Brave's language support is limited to the UI (menus, area local notations,
etc).  Brave's index only seems to support a locale, but it does not seem to
support any languages in its index.  The choice of available languages is very
small (and its not clear to me where the difference in UI is when switching
from en-us to en-ca or en-gb).

In the :py:obj:`EngineTraits object <searx.enginelib.traits.EngineTraits>` the
UI languages are stored in a custom field named ``ui_lang``:

.. code:: python

    "custom": {
      "ui_lang": {
        "ca": "ca",
        "de-DE": "de-de",
        "en-CA": "en-ca",
        "en-GB": "en-gb",
        "en-US": "en-us",
        "es": "es",
        "fr-CA": "fr-ca",
        "fr-FR": "fr-fr",
        "ja-JP": "ja-jp",
        "pt-BR": "pt-br",
        "sq-AL": "sq-al"
      }
    },

Implementations
===============

    N)	urlencodeurlparse)parser)html)locales)EngineTraits)SXNG_Response)EngineResults)eval_xpath_getindexeval_xpath_listextract_textget_embeded_stream_urljs_obj_str_to_json_strjs_obj_str_to_pythonzhttps://search.brave.com/	Q22906900FHTML)websitewikidata_idofficial_api_documentationuse_official_apirequire_api_keyresultssearch)r   videosimagesnewsgogglesbrave_category Goggles
   Tstrictmoderateoff)      r   pdpwpmpy)dayweekmonthyeartime_range_mapqueryparamsreturnc                    | dd}t         rd|d<   t        dv r`|j                  dd      dz
  r|j                  dd      dz
  |d<   t        j                  |d	         rt        j                  |d	         |d
<   t        dk(  r	t        |d<   d|d   d<   t
         t         dt        |       |d<   t        j                  d|d          t        j                  |d   d      |d   d<   d|d   d<   d|d   d<   t        j                  |d   d      }|j                  d      d   j                         |d   d<   t        j                  |d   t        j                   d   d      }||d   d<   t        j                  d |d          y )!Nweb)qsource1
spellcheckr   r   pagenor&   offset
time_rangetfr   
goggles_idzgzip, deflateheaderszAccept-Encoding?urlzurl %s
safesearchr$   cookies0useLocation
summarizersearxng_localeall-countryui_langzen-usz
cookies %s)brave_spellcheckr   getr/   r    base_urlr   loggerdebugsafesearch_maptraits
get_regionsplitlowerr   get_engine_localecustom)r0   r1   argsengine_regionrL   s        $/root/searxng/searx/engines/brave.pyrequestr\      s    D  \..::h"Q&#ZZ!4q8DNf\23'++F<,@ADJ"$\+:F9'(j 0)D/1BCF5M
LL6%=) '5&8&89Mu&UF9l#'*F9m$&)F9l#%%f-=&>FM#0#6#6s#;B#?#E#E#GF9i ''/?(@&--PYBZ\cdG#*F9i 
LLvi01    published_date_rawc                 f    | y 	 t        j                  |       S # t         j                  $ r Y y w xY w)N)r   parseParserError)r^   s    r[   _extract_published_daterb      s8    !||.// s    00textc                    | | j                  d      | j                  d       } | st        d      | j                  d      }| j                  d      }| || }d|z   dz   }t        |      }t	        j
                  |      }|S )Nz<scriptz</scriptz)can't find JS/JSON data in the given textzdata: [{z}}]{z}}]})index
ValueErrorrindexr   jsonloads)rc   startend
js_obj_strjson_strdatas         r[   extract_json_datarp      s     

9%

:(>?DDEEJJz"E
++e
CeCJz!F*J &j1H!ZZ1DKr]   respc                    t         dv rt        |       S t         dv rt        |       S t        | j                        }|d   d   d   d   d   }t         dk(  rt        |      S t         dk(  rt        |      S t        d	t                )
Nr9   r   ro   r&   bodyresponser   r   zUnsupported brave category: )r   _parse_search_parse_newsrp   rc   _parse_images_parse_videosrg   )rq   	json_data	json_resps      r[   rt   rt     s    ..T""&!4   #4DII">I"+F"3A"6v">v"Fz"RI!Y''!Y''
3N3CD
EEr]   c           	      T   t               }t        j                  | j                        }t	        |d      D ](  }t        |ddd       }t        |ddd       }||t        |      j                  s<d}d }t        |ddd      }t        |      rPt        |      }t        t        |ddd            }	|	r+t        |	      }|j                  |	      j                  d	      }t        |d
dd      }
|j                  j                  d|t        |      |||
      }|j                  |       t        |ddg       }t        |      st!        |      }|s||d<   d|d<   + t	        |d      D ]7  }|j#                  |j                  j                  dt        |      i             9 |S )Nz#//div[contains(@class, 'snippet ')]z
.//a/@hrefr   defaultz!.//div[contains(@class, 'title')]r   z7.//div[contains(concat(' ', @class, ' '), ' content ')]z(.//span[contains(@class, 't-secondary')]z- 
	z-.//a[contains(@class, 'thumbnail')]//img/@srcdefault.html)templaterA   titlecontentpublishedDate	thumbnailzA.//div[contains(@class, 'video-snippet') and @data-macro='video']
iframe_srcvideos.htmlr   z&//a[contains(@class, 'related-query')]
suggestion)r
   r   
fromstringrc   r   r   r   netloclenr   rb   lstripstriptypesLegacyResultaddr   append)rq   resdomresultrA   	title_tagr   pub_date_content	_pub_dater   item	video_tagr   r   s                  r[   ru   ru   !  s   
/C
//$))
$C!#'LM 31-flAtT'0SUV`de	;)+8C=3G3G
 'E	
 x="8,G$#H.XZ[eghI 29=!..399(C,V5dfgqst	yy%%#y)" & 
 	'O	
	 y> 04J%/\"#0Z g31j &c+ST U


399))<j9Q*RSTU Jr]   c           
      r   t               }t        j                  | j                        }t	        |d      D ]~  }t        |ddd       }|t	        |d      }t	        |d      }t        |ddd      }|j                  j                  d	|t        |      |t        |      
      }|j                  |        |S )Nz://div[contains(@class, 'results')]//div[@data-type='news']z-.//a[contains(@class, 'result-header')]/@hrefr   r|   z*.//span[contains(@class, 'snippet-title')]z.//p[contains(@class, 'desc')]z3.//div[contains(@class, 'image-wrapper')]//img/@srcr   r~   )r   rA   r   r   r   )
r
   r   r   rc   r   r   r   r   r   r   )	rq   r   r   r   rA   r   r   r   r   s	            r[   rv   rv   `  s    
/C
//$))
$C!#'cd !&*Y[\fjk;(TU!&*JK'0eghrtu	yy%%#u% ) & 
 	!$ Jr]   rz   c           
          t               }| d   D ]I  }|j                  j                  d|d   |d   |d   |d   d   |d   d   	      }|j                  |       K |S )
Nr   zimages.htmlrA   r   r6   
propertiesr   src)r   rA   r   r6   img_srcthumbnail_src)r
   r   r   r   )rz   r   r   r   s       r[   rw   rw   y  s}    
/CI& 	yy%%"u/(#<(/ -e4 & 
 		 Jr]   c                     t               }| d   D ]{  }|j                  j                  d|d   |d   |d   |d   d   |d   d   t        |d         	      }|d
   |d
   d   |d
<   t	        |d         }|r||d<   |j                  |       } |S )Nr   r   rA   r   descriptionvideodurationage)r   rA   r   r   lengthr   r   r   r   r   )r
   r   r   rb   r   r   )rz   r   r   r   r   s        r[   rx   rx     s    
/CI& yy%%"u/=)'?:.G_Z01&-@ & 
 +* &{ 3E :D+F5M:
!+D!$ Jr]   engine_traitsc           
         ddl }ddlm}m} ddlm} i | j                  d<   ddi} |dd	
      }|j                  st        d      t        j                  |j                        }|j                  d      D ]  }|j                  d      }		 |j                  j                  |	d      }
|
j                  r$ ||j                  j                  |	d            }n# ||j                  j                  |	d            }| j                  d   j                  |      }|r||	k7  rt#        d|d|d|	       |	| j                  d   |<     |dd	
      }|j                  st        d      |j                  |j                  j%                  d      t'        d      z   d }|d|j%                  d       }t)        |      }|j+                         D ]  \  }}|dk(  rd| _        |d   }|j.                  j1                  |d      D ]  }|j                  ||      }	  ||j                  j                  |d|j3                                     }| j4                  j                  |      }|r||k7  rt#        d|d|d|       || j4                  |<     y# |j                   $ r Y 3w xY w# |j                   $ r Y w xY w)z[Fetch :ref:`languages <brave languages>` and :ref:`regions <brave
    regions>` from Brave.r   N)language_tag
region_tag)rN   rL   nonbz!https://search.brave.com/settings   )timeoutz(Response from Brave languages is not OK.z+//section//option[@value='en-us']/../optionvaluerI   )sepzCONFLICT: babel z --> z, zQhttps://cdn.search.brave.com/serp/v2/_app/immutable/chunks/parameters.734c106a.jsz&Response from Brave regions is not OK.zoptions:{allzoptions:z},k={defaultrH   T)de_facto_)babel.languagessearx.localesr   r   searx.networkrN   rX   okRuntimeErrorr   r   rc   xpathLocaler`   	territoryUnknownLocaleErrorprintrf   r   r   items
all_locale	languagesget_official_languagesupperregions)r   babelr   r   rN   lang_maprq   r   optionrL   lsxng_tagconflict
country_jscountry_tagskvcountry_taglang_tags                      r[   fetch_traitsr     s    6!&(M#d|H 2A>D77EFF
//$))
$C))IJ <**W%	""7"4A{{%ell&8&8c&8&JK'(:(:7(:(LM
 !''	266x@7"8WUV4;Y'1#<* [D 77CDD499??>:S_LNOJ>j..~>?J'
3L""$ :1:',M$j >>{UY>Z 	:H||Hh7H%ell&8&8HkN_N_Na9b&cd %,,00:H{*XxQ\]^.9M!!(+	::1 '' 		F ++ s%   A0J4J.J+*J+.K ?K )4__doc__ri   typingturllib.parser   r   dateutilr   lxmlr   searxr   searx.enginelib.traitsr   searx.extended_typesr	   searx.result_typesr
   searx.utilsr   r   r   r   r   r   aboutrO   
categoriesr   Literal__annotations__r    strrM   pagingmax_pagerB   rR   time_range_supportr/   dictAnyr\   rb   rp   rt   ru   rv   rw   rx   r    r]   r[   <module>r      s  tl  
    / . ,  +"&	 '
MU		IJ U   0  
. 
*7 . 	"S#X "23 "2S!%%Z 0 "2T "2Jd
 C Daee$4 *F= F] F2< <- <~m  2T#quu*- - "T#quu*- - 0L: L:r]   