ij UdZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z ddl mZmZddlmZddlmZddlmZdd lmZmZmZmZdd lmZdd lmZdd lmZdd l m!Z!m"Z"m#Z#ddl m$Z$m%Z%ddl&m'Z'ddl(m)Z)m*Z*ddl+m,Z,ddl-m.Z.m/Z/ddl&m0Z0e0jbdZ0e2e!zZ3ejhe5d< e$e%zZ6ejhe5d<dZ7ejpdejrZ:ejpdejrZ;ejpdZ<ejpdZ=ejpdZ>ejpdZ?ejpdZ@ejpdZAejpd ZBd!ZCiZDeEe2e!fe5d"<iZFeEe2eEe2e2ffe5d#<Gd$d%ZGeGZHd&e2fd'ZIdfd(e2dzd&e2fd)ZJd&e2fd*ZKGd+d,eZLd-e2d&e2fd.ZMd/e2d&e2fd0ZN dgd1eOe6e6ze2ze zePzdzd2ePd&e2dzfd3ZQd4e2d5e2d&e2fd6ZRd1eOe6e6ze2ze zePzdzd5e2d&e2fd7ZSd8e ejejfd9eUe2d&e e2ejffd:ZVdhd;eWeXzdZZd?e2d&eWfd@Z[djdAe2dBe2dCe2dDe2d&e2f dEZ\dFeOe2e2zd&eWfdGZ]dHe2dIe2d&ejfdJZ_dKejd&e2fdLZ`dMe2d&e2fdNZadMe2fdOZbdPeEe2e2fd&e e2ge2ffdQZcdRe2d&eEe2eEe2e2fffdSZddTe3d&e!fdUZedVe6dTe3d&ejfdWZfdfdVe6dTe3dXeWdzd&eOejfdYZgeHfdVe6dTe3dZeWdDejd&ejf d[Zhd4e2fd\Zid]eje2d&e2fd^Zkd]eje2d&e2fd_Zld]eje2d&e2fd`Zmdae2d&ejfdbZndae2d&e2fdcZodde2d&edzfdeZpy)kz!Utility functions for the enginesN)MutableMappingCallable)Number)splitextjoin)choice) HTMLParser)escape)urljoinurlparseparse_qs urlencode) timedelta) MarkdownIt)html)XPath XPathErrorXPathSyntaxError) ElementBase_Elementsettings) USER_AGENTSgsa_useragents_loader) VERSION_TAG)SearxXPathSyntaxExceptionSearxEngineXPathException)loggerutils XPathSpecType ElementType)scriptstylez%u([0-9a-fA-F]{4})z%([0-9a-fA-F]{2})z(["\'`])z ([\{\s,])([\$_\w][\$_\w0-9]*)(:)z)void\s+[0-9]+|void\s*\([0-9]+\)|undefinedz(([\[\,:])\s*(\-?)\s*([0-9_]*)\.([0-9_]*)z([\[\,:])\s*(\-?)\s*([0-9_]+)z\s*,\s*([\]\}])z\\(.)z"\bfnrtu _XPATH_CACHE_LANG_TO_LC_CACHEceZdZdZy) _NotSetClasszInternal class for this module, do not create instance of this class. Replace the None value, allow explicitly pass None as a function argumentN)__name__ __module__ __qualname____doc__/root/searxng/searx/utils.pyr'r'<sQr-r'returncJdtdtddjS)zReturn the SearXNG User AgentzSearXNG/ outgoinguseragent_suffix)rrstripr,r-r.searxng_useragentr5Ds* k]!HZ$89K$L#M N T T VVr- os_stringc~tdj|xsttdttdS)zKReturn a random browser User Agent See searx/data/useragents.json uaosversions)r9version)rformatr)r6s r. gen_useragentr=Is@ t  # #  1{401{:./ $ r-c.ttdzS)zjReturn a random "Google Go App" User Agent suitable for Google See searx/data/gsa_useragents.txt z NSTNWV)rrr,r-r.gen_gsa_useragentr?Ts ') *Y 66r-ceZdZdZdZdedeeeedzfddfdZdeddfdZ d Z d eddfd Z d eddfd Z d eddfdZ dZdeddfdZy)HTMLTextExtractorz(Internal class to extract text from HTMLcJtj|g|_g|_yN)r __init__resulttagsselfs r.rDzHTMLTextExtractor.__init___sD!!# ! r-tagattrsNr/c||jj||dk(r|jjdyy)Nbrr1)rFappendrE)rHrIrJs r.handle_starttagz!HTMLTextExtractor.handle_starttagds2  $; KK  s # r-c|jsy||jdk7r |jjd|dy|jjy)Nz)rFrErMpop)rHrIs r. handle_endtagzHTMLTextExtractor.handle_endtagisFyy  $))B-  KK  C5{ +  r-cJ|j xs|jdtvS)NrP)rF _BLOCKED_TAGSrGs r. is_valid_tagzHTMLTextExtractor.is_valid_tagss 99}B " ] BBr-datac\|jsy|jj|yrCrVrErM)rHrWs r. handle_datazHTMLTextExtractor.handle_datavs#  "  4 r-namec|jsy|ddvrt|ddd}n t|}|jjt |y)Nr)xX)rVintrErMchr)rHr[ codepoints r.handle_charrefz HTMLTextExtractor.handle_charref{sN  "  7j DHb)ID I 3y>*r-c\|jsy|jj|yrCrY)rHr[s r.handle_entityrefz"HTMLTextExtractor.handle_entityrefs%  "  4 r-cTdj|jjS)N)rrEr4rGs r.get_textzHTMLTextExtractor.get_textswwt{{#))++r-messagect|rC)AssertionError)rHrjs r.errorzHTMLTextExtractor.errorsW%%r-)r(r)r*r+rDstrlisttuplerNrSrVrZrdrfrirmr,r-r.rArA\s2" $3$tE#sTz/4J/K$PT$ C!!! +3+4+!S!T!,&S&T&r-rAhtml_strc|sy|jddjdd}dj|j}t} |j ||j |jS#t $rHt}|j t|d|j Y|jSwxYw)aRExtract text from a HTML string Args: * html_str (str): string HTML Returns: * str: extracted text Examples: >>> html_to_text('Example #2') 'Example #2' >>> html_to_text('Example') 'Example' >>> html_to_text(r'regexp: (?<![a-zA-Z]') 'regexp: (?>> html_to_text(r'

Lorem ipsum dolor sit amet

') 'Lorem ipsum dolor sit amet

' >>> html_to_text(r'> < a') '> < a' rh r1 T)quote) replacersplitrAfeedcloserlr ri)rqss r. html_to_textr{s4 c*224=Hxx()HA x  ::<    vhd+,  ::< s!B?CC markdown_strcttdddijddgj|}t|S)a-Extract text from a Markdown string Args: * markdown_str (str): string Markdown Returns: * str: extracted text Examples: >>> markdown_to_text('[example](https://example.com)') 'example' >>> markdown_to_text('## Headline') 'Headline' commonmark typographerT replacements smartquotes)renablerenderr{)r|rqs r.markdown_to_textrsB$ <-!67>>P]?^_ffgst   !!r- xpath_results allow_nonect|tr+d}|D]}|t|xsdz}|jSt|trXt j |ddd}|jjdd}dj|jSt|tttfr t|S||ry| |s td td ) aiExtract text from a lxml result - If ``xpath_results`` is a list of :py:obj:`ElementType` objects, extract the text from each result and concatenate the list in a string. - If ``xpath_results`` is a :py:obj:`ElementType` object, extract all the text node from it ( :py:obj:`lxml.html.tostring`, ``method="text"`` ) - If ``xpath_results`` is of type :py:obj:`str` or :py:obj:`Number`, :py:obj:`bool` the string value is returned. - If ``xpath_results`` is of type ``None`` a :py:obj:`ValueError` is raised, except ``allow_none`` is ``True`` where ``None`` is returned. rhunicodetextF)encodingmethod with_tailrsr1Nz$extract_text(None, allow_none=False)zunsupported type) isinstancero extract_textr4r!rtostringrvrrwrnrbool ValueError)rrrEers r.rrs&-& 6A|A4"5F 6||~--MM    zz|##D#.xx %%-#vt!45=!!Z?@@ ' ((r-urlbase_urlc@|jdr,t|}dj|jxsd|}n|jdr t ||}d|vr t ||}t|}|j s t d|js|dz }|S)a\Normalize URL: add protocol, join URL with base_url, add trailing slash if there is no path Args: * url (str): Relative URL * base_url (str): Base URL, it must be an absolute URL. Example: >>> normalize_url('https://example.com', 'http://example.com/') 'https://example.com/' >>> normalize_url('//example.com', 'http://example.com/') 'http://example.com/' >>> normalize_url('//example.com', 'https://example.com/') 'https://example.com/' >>> normalize_url('/path?a=1', 'https://example.com') 'https://example.com/path?a=1' >>> normalize_url('', 'https://example.com') 'https://example.com/' >>> normalize_url('/test', '/path') raise ValueError Raises: * lxml.etree.ParserError Returns: * str: normalized URL z//z{0}:{1}http/z://zCannot parse url) startswithr r<schemer netlocrpath)rrparsed_search_url parsed_urls r. normalize_urlrs6 ~~d$X.077A63G  h$ Ch$#J   +,, ?? s  Jr-cj|gk(r tdt|}|r t||Std)aExtract and normalize URL from lxml Element Example: >>> def f(s, search_url): >>> return searx.utils.extract_url(html.fromstring(s), search_url) >>> f('https://example.com', 'http://example.com/') 'https://example.com/' >>> f('https://example.com', 'http://example.com/') 'https://example.com/' >>> f('//example.com', 'http://example.com/') 'http://example.com/' >>> f('//example.com', 'https://example.com/') 'https://example.com/' >>> f('/path?a=1', 'https://example.com') 'https://example.com/path?a=1' >>> f('', 'https://example.com') raise lxml.etree.ParserError >>> searx.utils.extract_url([], 'https://example.com') raise ValueError Raises: * ValueError * lxml.etree.ParserError Returns: * str: normalized URL zEmpty url resultsetz URL not found)rrr)rrrs r. extract_urlr2s=8.// } %C S(++ _ %%r- dictionary propertiesc>|Dcic] }||vs|||c}Scc}w)zExtract a subset of a dict Examples: >>> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'C']) {'A': 'a', 'C': 'c'} >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D']) {'A': 'a'} r,)rrks r. dict_subsetrWs''1 DAOAz!}  DD Ds size precisioncxgd}t|}d}|dkDr||kr|dz }|dz }|dkDr||krd||||fzS)zGDetermine the *human readable* value of bytes on 1024 base (1KB=1024B).)zB KBMBGBTBrir_g@z%.*f %slenrrrzr]ps r.humanize_bytesrcs[&A AA A +!a% Qf} +!a%  41. ..r-cxgd}t|}d}|dkDr||kr|dz }|dz }|dkDr||krd||||fzS)z9Determine the *human readable* value of a decimal number.)rhKMBTrir_g@@z%.*f%srrs r.humanize_numberros[ A AA A +!a% Qf} +!a% y$!- --r- number_strc:|jr t|Sy)z=Convert number_str to int or 0 if number_str is not a number.r)isdigitra)rs r.convert_str_to_intr{s: r-txtbeginenddefaultc |j|t|z}|||j||S#t$r|cYSwxYw)aExtract the string between ``begin`` and ``end`` from ``txt`` :param txt: String to search in :param begin: First string to be searched for :param end: Second string to be searched for after ``begin`` :param default: Default value if one of ``begin`` or ``end`` is not found. Defaults to an empty string. :return: The string between the two search-strings ``begin`` and ``end``. If at least one of ``begin`` or ``end`` is not found, the value of ``default`` is returned. Examples: >>> extr("abcde", "a", "e") "bcd" >>> extr("abcde", "a", "z", deafult="nothing") "nothing" )indexrr)rrrrfirsts r.extrrsJ, % 3u:-5399S%011 s14 AAnumc`t|trt|dkry|d}t|S)zConvert num to int or 0. num can be either a str or a list. If num is a list, the first element is converted to int (or return 0 if the list is empty). If num is a str, see convert_str_to_int r_r)rrorr)rs r. int_or_zerors0 #t s8a<!f c ""r-filename module_dircFt|d}t||}tjj ||}|st d|dtjj |}|jst d|d|jj||S)NrzError loading 'z' module) rr importlibutilspec_from_file_locationrmodule_from_specloader exec_module)rrmodnamemodpathspecmodules r. load_modulersx #G:x(G >> 1 1'7 CD ?7)8<== ^^ , ,T 2F ;;?7)8<==KKF# Mr-objcjt|tr|St|dr t|St|S)z)Convert obj to its string representation.__str__)rrnhasattrrepr)rs r. to_stringrs.#s sI3x 9r-stringcbtjd|}tjd|}|S)uPython implementation of the unescape javascript function https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape Examples: >>> ecma_unescape('%u5409') '吉' >>> ecma_unescape('%20') ' ' >>> ecma_unescape('%F3') 'ó' cJtt|jddSNr_r`rbragrouprs r.zecma_unescape..c#aggaj"2E.Fr-cJtt|jddSrrrs r.rzecma_unescape..rr-)_ECMA_UNESCAPE4_REsub_ECMA_UNESCAPE2_RE)rs r. ecma_unescapers/ # #$F OF  # #$F OF Mr-cd}g}|D]3}t|tfd|Dr#|j|5dj|S)zRemoves unicode's "PRIVATE USE CHARACTER"s (PUA_) from a string. .. _PUA: https://en.wikipedia.org/wiki/Private_Use_Areas ))ii)ii)iic3DK|]\}}|cxkxr|kncywrCr,).0abis r. z&remove_pua_from_str..s4v1qA{{{4s rh)ordanyrMr)r pua_rangesrzcrs @r.remove_pua_from_strrsT NJA  F 44 4    771:r-replacesc|jDcic]\}}tj||c}}tjdj j dt ffd }|Scc}}w)N|rc.jfd|S)NcRtj|jdS)Nr)rer r)mreps r.rz.func..sS1771:)>%?r-)r)rpatternrs r.funcz*get_string_replaces_function..funcs{{?FFr-)itemsrr compilerkeysrn)rrvrrrs @@r.get_string_replaces_functionrsa'/~~'7 8tq!299Q<? 8Cjj#((*-.GG3G K 9s A;r[cXdtvriStdD]}d|vr||dk(s|cSiS)zDReturn engine configuration from settings.yml of a given engine nameenginesr[r)r[engines r.get_engine_from_settingsrsH  9%    6&> !M  Ir- xpath_specct|tr/tj|d}| t |}|t|<|St|tr|Std#t $r%}t |t|j|d}~wwxYw)a'Return cached compiled :py:obj:`lxml.etree.XPath` object. ``TypeError``: Raised when ``xpath_spec`` is neither a :py:obj:`str` nor a :py:obj:`lxml.etree.XPath`. ``SearxXPathSyntaxException``: Raised when there is a syntax error in the *XPath* selector (``str``). Nz5xpath_spec must be either a str or a lxml.etree.XPath) rrnr$getrrrmsg TypeError)rrErs r. get_xpathr s*c"!!*d3 > Oz*(.L $ *e$ K LL$ O/ CJGQN Os A B & BB elementc t|} ||S#t$rJ}dj|jDcgc] }t |ncc}wc}}t |||d}~wwxYw)aEquivalent of ``element.xpath(xpath_str)`` but compile ``xpath_str`` into a :py:obj:`lxml.etree.XPath` object once for all. The return value of ``xpath(..)`` is complex, read `XPath return values`_ for more details. .. _XPath return values: https://lxml.de/xpathxslt.html#xpath-return-values ``TypeError``: Raised when ``xpath_spec`` is neither a :py:obj:`str` nor a :py:obj:`lxml.etree.XPath`. ``SearxXPathSyntaxException``: Raised when there is a syntax error in the *XPath* selector (``str``). ``SearxEngineXPathException:`` Raised when the XPath can't be evaluated (masked :py:obj:`lxml.etree..XPathError`). r1N)r rrargsrnr)rrxpathrrargs r. eval_xpathrsc&Z(E@W~ @hh/1A//0' C8a?@s A(A#A A##A(min_lenct||}t|ts t|d|&|t |kDrt|dt |z|S)zSame as :py:obj:`searx.utils.eval_xpath`, but additionally ensures the return value is a :py:obj:`list`. The minimum length of the list is also checked (if ``min_len`` is set).zthe result is not a listzlen(xpath_str) < )rrrorrrn)rrrrEs r.eval_xpath_listr5sU %Wj9F fd #' 4NOOwV4' 4G#g,4VWW Mr-rct||}t| |cxkrt|kr||S|tk(rt|dt |zdz|S)aDSame as :py:obj:`searx.utils.eval_xpath_list`, but returns item on position ``index`` from the list (index starts with ``0``). The exceptions known from :py:obj:`searx.utils.eval_xpath` are thrown. If a default is specified, this is returned if an element at position ``index`` could not be determined. zindex z not found)rr_NOTSETrrn)rrrrrEs r.eval_xpath_getindexrBs`Wj 1F F |u*s6{*e}+'( Hs5z4IL4XYY Nr-c4t|}d}|jdvrL|jdk(r=|jr1t |jj dg}|rd|dz}|S|jdvrt d|i}d |z}|S|jd vrD|jjd r)|jjd r|d z}|S|dz}|S|jdvrP|jjdr5d|jvr'|jjd}|d}d|z}|S|jdvrP|jjdr5|jjd }t|dk(r |d}d|z}|S|jdvrr|jjdrW|jjd }|d}d}|jdr|dd}d}n|jdrd}d|d|d}|S) z Converts a standard video URL into its embed format. Supported services include Youtube, Facebook, Instagram, TikTok, Dailymotion, and Bilibili. N)zwww.youtube.comz youtube.comz/watchrz'https://www.youtube-nocookie.com/embed/r)zwww.facebook.comz facebook.comhrefz@https://www.facebook.com/plugins/video.php?allowfullscreen=true&)zwww.instagram.comz instagram.comz/p/rembedz/embed)zwww.tiktok.comz tiktok.comz/@z/video/r_zhttps://www.tiktok.com/embed/)zwww.dailymotion.comzdailymotion.comz(https://www.dailymotion.com/embed/video/)zwww.bilibili.comz bilibili.comavaidBVbvidz(https://player.bilibili.com/player.html?=z(&high_quality=1&autoplay=false&danmaku=0) r rrqueryr r rrendswithrwr)rr iframe_srcvideo_id encoded_href path_parts param_keys r.get_embeded_stream_urlr+ZsT #JJ>>:??V^C^cmcscsJ,,-11#r: BXa[PJ^ Y   B B &#/ WZff T O   D DIcIcdiIj ?? # #C (wJJ GxJF ? == OO & &t ,  (__**95 a=4x? 2 -   H HZ__MgMghqMr__**3/ z?a !!}HChNJ$    B BzGaGabkGl__**3/ a=   t $|HI   &I7yk8*Lt u  r-matchc|jdxs|jd}|tvrd|S|dk(rdS|dk(rdS|S)Nr_r\r]z\u00rsrh)r_JSON_PASSTHROUGH_ESCAPES)r,_escapes r._j2p_process_escaper1sckk!n. AG / / WI 3W3:T/BHOr-c|jd|jdz|jdjddxsdzdz|jdjddxsdzS) Nr_rr_rh0.rrvr,s r. _j2p_decimalr9sr A ++a.  ;;q> ! !#r * 1c 3   ;;q> ! !#r * 1c  3r-c|jd|jdz|jdjddzS)Nr_rrr3rhr7r8s r. _j2p_decimal2r;s7 ;;q>EKKN *U[[^-C-CC-L LLr- js_obj_strct|}|dk(r td tj|S#tj$r'}t j d|td|d}~wwxYw)zConvert a javascript variable into JSON and then load the value It does not deal with all cases, but it is good enough for now. chompjs has a better implementation. rh#js_obj_str can't be an empty stringz=Internal error: js_obj_str_to_python creates invalid JSON: %sz)js_obj_str_to_python creates invalid JSONN)js_obj_str_to_json_strrjsonloadsJSONDecodeErrorrdebug)r<rzrs r.js_obj_str_to_pythonrDsk z*ABw>??Mzz!}   M UWXYDE1LMs2A,"A''A,cpt|ts td|dk(r tdd}tj |}d}t |D]\}}||k(r |sd}d||<n|rm|j dtd}tjt|}|dk(r|j dd }|||<|r|dddk(r||dz dd ||dz <nv| |d vr|}d||<nh|ftjd |}tjt|}tjt|}t jd |}|||<t#|dkDxr|d dk(}dj%|}t&jd|}|j tddj)}|S)Nzjs_obj_str must be of type strrhr>F":r_'z\"rP)rFrH`nullc$|jdS)Nr_)rr8s r.rz(js_obj_str_to_json_str..sEKKNr-rr.z\1"\2"\3)rrnr_JS_STRING_DELIMITERSrw enumeratervrb_JS_STRING_ESCAPE_RErr1_JS_VOID_OR_UNDEFINED_RE_JS_DECIMAL_REr9_JS_DECIMAL2_REr;_JS_EXTRA_COMA_RErr_JS_QUOTE_KEYS_REr4)r< in_stringpartsblackslash_just_beforerrrzs r.r?r?s j# &9::R>??I " ' ' 3E"% 9>1 >"8IE!H  #s1v&A$(()x A k1-A #a&#$$&A Hr- duration_strc|j}|sy dg|jdzdd}tt|\}}}t |||S#t t f$rYywxYw)zParse a time string in format MM:SS or HH:MM:SS and convert it to a `timedelta` object. Returns None if the provided string doesn't match any of the formats. N00rGr)hoursminutesseconds)r4rwmaprarrr )rW time_partsrZr[r\s r.parse_duration_stringr_sw  %%'L  f|11#66; "%c:"6wugwGG  "   s9AA! A!rC)F)r)r)rh)qr+rrimportlib.utilr@typestypingtcollections.abcrrnumbersros.pathrrrandomr html.parserr rr urllib.parser r r rdatetimer markdown_itrlxml lxml.etreerrrrrsearxr searx.datarr searx.versionrsearx.exceptionsrrrgetChildrnr TypeAlias__annotations__r!rUrUNICODErrrLrSrOrPrQrRrNr/r$dictr%r'rr5r=r?rAr{rrorrrrAnysetrrafloatrrrrr ModuleTyperrrrrrr rrrr+Matchr1r9r;rDr?r_r,r-r.r|sQ'  4""??"::,9%Q  ! 5[ q{{(?'1 Q[[1$ RZZ 5rzzBRZZ 4bjjA" ;/BJJBC%2::&RSGH"**=>BJJ12!rzz(+'!# d3:#/14T#s(^+,1QQ .W3W S4Z37375& 5&p&3&3&R"3"3"2)) ${2S86ADH4O)))) 4Z))X/s/c/c/d"&tK0;>DvMPTTW[["&gj"&or"&J EN155!%%<8 Ec#h ESabeghglglblSm E /u / / .#+ .# . 3 3 c#C#s: #T#Y_ # # # 3 53C3C 155S##*  4S>hucz>R 3 4T#s(^0C+D M-MEM4@ @@155@8 [ m cTXj dhijinindo "   UU  UU 0;;|rxx} #M#M3MMSMQUUM"T sT sT n D0@r-