
    i)                     ,   d Z ddlmZ ddlmZ ddlmZ ddlmZm	Z	 ddl
mZ dZ	 dZ	 g Z	 dZ	 d	Z	 d
Z	 i Z	 i Z	 dZ	 dZ	 dZ	 d
Z	 dZ	 d
Z	 dZ	 dZ	 dZ	 d
Z	 d
Z	 dZ	 dZ 	 dZ!	 dZ"	 dddddZ#	 dZ$	 ddddZ%	 d Z&d Z'd Z(d Z)d Z*d Z+d Z,d Z-d Z.y) ac  The JSON engine is a *generic* engine with which it is possible to configure
engines in the settings.

Configuration
=============

Request:

- :py:obj:`search_url`
- :py:obj:`lang_all`
- :py:obj:`soft_max_redirects`
- :py:obj:`method`
- :py:obj:`request_body`
- :py:obj:`cookies`
- :py:obj:`headers`

Paging:

- :py:obj:`paging`
- :py:obj:`page_size`
- :py:obj:`first_page_num`

Time Range:

- :py:obj:`time_range_support`
- :py:obj:`time_range_url`
- :py:obj:`time_range_map`

Safe-Search:

- :py:obj:`safe_search_support`
- :py:obj:`safe_search_map`

Response:

- :py:obj:`title_html_to_text`
- :py:obj:`content_html_to_text`
- :py:obj:`no_result_for_http_status`

JSON query:

- :py:obj:`results_query`
- :py:obj:`url_query`
- :py:obj:`url_prefix`
- :py:obj:`title_query`
- :py:obj:`content_query`
- :py:obj:`thumbnail_query`
- :py:obj:`thumbnail_prefix`
- :py:obj:`suggestion_query`


Example
=======

Here is a simple example of a JSON engine configure in the :ref:`settings
engines` section, further read :ref:`engines-dev`.

.. code:: yaml

  - name : mdn
    engine : json_engine
    paging : True
    search_url : https://developer.mozilla.org/api/v1/search?q={query}&page={pageno}
    results_query : documents
    url_query : mdn_url
    url_prefix : https://developer.mozilla.org
    title_query : title
    content_query : summary

Implementations
===============

    )Iterable)loads)	urlencode)	to_stringhtml_to_text)raise_for_httperrorNenGET F   z&hours={time_range_val}      i  i8"  )dayweekmonthyearz&filter=nonez&filter=moderatez&filter=strict)r   r      c              #      K   t        | t              r| j                         }nt        |       }|D ]  \  }}t	        |      |f  y wN)
isinstancedictitems	enumeratestr)iterabler   indexvalues       */root/searxng/searx/engines/json_engine.pyiterater      sJ     (D!  (#  u%j% s   AA	c                 D    t        | t              ryt        | t              S )NF)r   r   r   )objs    r   is_iterabler"   
  s    #sc8$$    c                 d    g }| j                  d      D ]  }|dk(  r	|j                  |        |S )N/r   )splitappend)queryqparts      r   parser+     s;    
AC  2:	 Hr#   c           	      r   g }|s|S |d   }t        |       D ]  \  }}t        |      dk(  r?||k(  r|j                  |       +t        |      s7|j	                  t        ||             St        |      s_||k(  r|j	                  t        ||dd               |j	                  t        ||              |S )Nr   r   )r   lenr'   r"   extenddo_query)datar)   retqkeykeyr   s         r   r/   r/     s    
C
Q4Ddm /
Uq6Q;d{

5!U#

8E1-.u%d{

8E1QR512

8E1-./ Jr#   c                 0    t        |      }t        | |      S r   )r+   r/   )r0   query_stringr)   s      r   r(   r(   1  s    lAD!r#   c                 J   t         }|d   dk7  r|d   dd }d}|j                  d      r:t        j                  |j                  d            }t        j	                  |      }d}|d   rt
        |d      }t        d	| i      dd ||d
   dz
  t        z  t        z   ||d}|d   j                  t               |d   j                  t               t        j                  di ||d<   t        |d<   t        r| |d<   t        j                  di ||d<   t        |d<   d|d<   |S )z5Build request parameters (see :ref:`engine request`).languageallNr   r   
time_range)time_range_val
safesearchr)   pagenor   )r(   langr<   r9   safe_searchcookiesheadersurlmethodr(   r0   soft_max_redirectsFr    )lang_allgettime_range_maptime_range_urlformatsafe_search_mapr   	page_sizefirst_page_numupdater?   r@   
search_urlrB   request_bodyrC   )r(   paramsr=   r9   r:   r>   fps          r   requestrR   7  sA   DjU"j!"1%Jzz,'++FJJ|,DE#**.*I
Kl%f\&:; C<(,(#a'94~E "
B 9W%
9W%%%++F5MF87%,,2r2v#5F $)F !Mr#   c                     | S r   rD   )args    r   identityrU   _  s    Jr#   c                    t         rt        nt        }t        rt        nt        }i }	 t	        | t
              d   }t        t        |      z   |d<   t	        | t              d   } |t        |            |d<   	 t	        | t              d   } |t        |            |d<   	 t        r(t	        | t              d   }t        t        |      z   |d<   |S #  Y y xY w#  d|d<   Y AxY w#  Y |S xY w)Nr   rA   titlecontentr   	thumbnail)title_html_to_textr   rU   content_html_to_textr(   	url_query
url_prefixr   title_querycontent_querythumbnail_querythumbnail_prefix)resulttitle_filtercontent_filter
tmp_resultrA   rW   rX   thumbnail_query_results           r   extract_response_inforg   c  s    #5<8L%9\xNJFI&q)&37
5fk*1-*9U+;<
7#.q1 .y/A B
9%*6?%CA%F"&6CY9Z&ZJ{# !
# "
9s$   AC 8'C  .C# CC #C(c                    g }t         r| j                  t         v r|S t        |        | j                  s|S t	        | j                        }dt
        v }t        rt        |t              }|s|S |d   }n|}|D ](  }t        |      }|s|rd|d<   |j                  |       * t        s|S t        |t              D ]  }|j                  d|i        |S )z<Scrap *results* from the response (see :ref:`result types`).onionsr   Tis_onion
suggestion)no_result_for_http_statusstatus_coder   textr   
categoriesresults_queryr(   rg   r'   suggestion_query)respresultsjsonrj   rsrb   re   rk   s           r   responserv     s    G T%5%59R%R99D:%H4'NU #*62
%)Jz"z"# D"23 3
j123Nr#   )/__doc__collections.abcr   rt   r   urllib.parser   searx.utilsr   r   searx.networkr   rN   rE   rl   rC   rB   rO   r?   r@   pagingrK   rL   rp   r\   r]   r^   r_   r`   ra   rq   rZ   r[   time_range_supportrH   rG   safe_search_supportrJ   r   r"   r+   r/   r(   rR   rU   rg   rv   rD   r#   r   <module>r      sf  HT %  " / -
@      N	 @& 0 F 
 -	  0 	 d
 0 f h j  6  k  +  -  (* 	
   "$);@PQ
 %0%P>%r#   