
    iB$                         d Z ddlmZ ddlmZ ddlmZmZmZm	Z	 ddl
mZ ddlmZ dZ	 dZ	 g Z	 dZ	 d	Z	 dZ	 dZ	 dZ	 d
Z	 d	Z	 d	Zd	Zi Z	 i Z	 dZ	 d	Z	 d
Z	 dZ	 dZ 	 d
Z!	 dZ"	 dddddZ#	 d
Z$	 ddddZ%	 d Z&defdZ'y)ad  The XPath engine is a *generic* engine with which it is possible to configure
engines in the settings.

.. _XPath selector: https://quickref.me/xpath.html#xpath-selectors

Configuration
=============

Request:

- :py:obj:`search_url`
- :py:obj:`lang_all`
- :py:obj:`soft_max_redirects`
- :py:obj:`method`
- :py:obj:`request_body`
- :py:obj:`cookies`
- :py:obj:`headers`

Paging:

- :py:obj:`paging`
- :py:obj:`page_size`
- :py:obj:`first_page_num`

Time Range:

- :py:obj:`time_range_support`
- :py:obj:`time_range_url`
- :py:obj:`time_range_map`

Safe-Search:

- :py:obj:`safe_search_support`
- :py:obj:`safe_search_map`

Response:

- :py:obj:`no_result_for_http_status`

`XPath selector`_:

- :py:obj:`results_xpath`
- :py:obj:`url_xpath`
- :py:obj:`title_xpath`
- :py:obj:`content_xpath`
- :py:obj:`thumbnail_xpath`
- :py:obj:`suggestion_xpath`


Example
=======

Here is a simple example of a XPath engine configured in the :ref:`settings
engines` section, further read :ref:`engines-dev`.

.. code:: yaml

  - name : bitbucket
    engine : xpath
    paging : True
    search_url : https://bitbucket.org/repo/all/{pageno}?name={query}
    url_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]/@href
    title_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]
    content_xpath : //article[@class="repo-summary"]/p

Implementations
===============

    )	urlencode)html)extract_textextract_url
eval_xpatheval_xpath_list)raise_for_httperror)EngineResultsNen FGET   z&hours={time_range_val}      i  i8"  )dayweekmonthyearz&filter=nonez&filter=moderatez&filter=strict)r   r      c                 `   t         }|d   dk7  r|d   dd }d}|j                  d      r:t        j                  |j                  d            }t        j	                  |      }d}|j                  d      }|	t
        |   }t        d	| i      dd ||d
   dz
  t        z  t        z   ||d}|d   j                  t               |d   j                  t               t        j                  di ||d<   t        |d<   t        r| |d<   t        j                  di ||d<   t        |d<   d|d<   |S )z5Build request parameters (see :ref:`engine request`).languageallNr   r   
time_range)time_range_val
safesearchqpagenor   )querylangr   r   safe_searchcookiesheadersurlmethodr   datasoft_max_redirectsFr	    )lang_allgettime_range_maptime_range_urlformatsafe_search_mapr   	page_sizefirst_page_numupdater!   r"   
search_urlr$   request_bodyr&   )r   paramsr   r   r   r    safe_search_valfargss           $/root/searxng/searx/engines/xpath.pyrequestr7      sH   DjU"j!"1%Jzz,'++FJJ|,DE#**.*I
Kjj.O"%o6 C<(,(#a'94~E "E 9W%
9W%%%..F5MF8g%,,5u5v#5F $)F !M    returnc                    t               }t        r| j                  t        v r|S t        |        | j                  s|S t        j                  | j                        }dt        v }t        rt        |t              D ]  }t        t        |t        d      t              }t        t        |t        d            }t        t        |t                    }|||d}t         r1t        |t               }	t#        |	      dkD  rt        |	t              |d<   t$        r%t&        t        t        |t$        d            z   |d<   |rd|d	<   |j)                  |        nt$        rt+        d
 t        |t              D        t-        t        t        |t                    t-        t        t        |t                    t-        t        t        |t$                          D ]%  \  }}}}
|j)                  |||t&        |
z   |d       ' nyt+        d t        |t              D        t-        t        t        |t                    t-        t        t        |t                          D ]  \  }}}|j)                  ||||d        t.        r1t1        |t.              D ]  }|j)                  dt        |      i         t2        j5                  dt#        |             |S )z<Scrap *results* from the response (see :ref:`result types`).onionsr   )min_len)r#   titlecontentr   	thumbnail
cached_urlTis_onionc              3   <   K   | ]  }t        |t                y wNr   r1   .0xs     r6   	<genexpr>zresponse.<locals>.<genexpr>3       UQ
+U   )r#   r=   r>   r@   rA   c              3   <   K   | ]  }t        |t                y wrC   rD   rE   s     r6   rH   zresponse.<locals>.<genexpr>C  rI   rJ   )r#   r=   r>   rA   
suggestionzfound %s results)r
   no_result_for_http_statusstatus_coder	   textr   
fromstring
categoriesresults_xpathr   r   	url_xpathr1   r   title_xpathcontent_xpaththumbnail_xpathlencached_xpathr@   appendzipmapsuggestion_xpathr   loggerdebug)respresultsdomrA   resultr#   r=   r>   
tmp_resultthumbnail_xpath_resultcachedrL   s               r6   responserf   
  s_   oG T%5%59R%R99
//$))
$C:%H%c=9 	'FofiKZXC a!PQE"?6=#IJG!$uIJ )8)Q&-.2.9:PR\.]J{+ +5_U[]istEu8v+v
<()-
:&NN:&)	'. /2U_S)5TUL/#{"CDL/#}"EFL/#|"DE	0 +UGV "!&#*&06&9$,  (+U_S)5TUL/#{"CDL/#}"EF( g#UG
 sUw\defg $S*:; 	EJNNL,z*BCD	E LL#S\2Nr8   )(__doc__urllib.parser   lxmlr   searx.utilsr   r   r   r   searx.networkr	   searx.result_typesr
   r1   r(   rM   r&   rR   rS   rU   rT   rV   r\   rX   r@   r!   r"   r$   r2   pagingr.   r/   time_range_supportr+   r*   safe_search_supportr-   r7   rf   r'   r8   r6   <module>rp      s1  DL #  N N - ,
@      N 4	 , 0 . 2  3

0  
 @ 
 -	  0  (* 	
   "$);@PQ
&RDm Dr8   