i0dZddlZdgZddlZddlmZddlmZm Z m Z m Z ddl m Z ddlmZmZddlmZej(rdd lmZeeeeeefZej6Gd dZed k(r%eZej?D] Z e!e  yy) zFSimple implementation to store TrackerPatterns data in a SQL database.NTrackerPatternsDB)Iterator)urlparse urlunparse parse_qsl urlencode) HTTPError) get_cachelog)get) CacheRowTypeceZdZdZgdZGddZdZdZdZde fd Z d e e fd Z d e e fd Z d ed eezfdZy)rdata_tracker_patterns)z-https://rules1.clearurls.xyz/data.minify.jsonz-https://rules2.clearurls.xyz/data.minify.jsonzQhttps://raw.githubusercontent.com/ClearURLs/Rules/refs/heads/master/data.min.jsoncteZdZUdZej ed<dZej ed<dZej ed<y)TrackerPatternsDB.Fieldsr url_regexp url_ignoredel_argsN) __name__ __module__ __qualname__rtFinal__annotations__rr,/root/searxng/searx/data/tracker_patterns.pyFieldsr&s, AGG AGG!''rr c"t|_y)N)r cacheselfs r__init__zTrackerPatternsDB.__init__,s [ rc|jjddk7r7|jjjdd|jyy)Nztracker_patterns loadedOK)r" propertiessetloadr#s rinitzTrackerPatternsDB.init/sD :: !: ;t C JJ ! ! % %&? F IIK Drchtjdg}|jD]a}||jj}||jj ||jj f}|j||dfc|jj||jy)Nz init searx.data.TRACKER_PATTERNSctx) r debugiter_clear_listr rrrappendr"setmanyctx_name)r$rowsrulekeyvalues rr*zTrackerPatternsDB.load8s 45%'((* ,Dt{{--.CT[[++,T[[))*E KKeT* +  , 4T]]3rr5c||jj}||jj||jjf}|jj |||j dy)N)r6r7r.expire)r rrrr"r)r3)r$r5r6r7s raddzTrackerPatternsDB.addFs_4;;))* '' ( %% &  3etLrreturnc#K|j|jj|jD]\}}||d|dfyw)Nr-rr)r+r"pairsr3)r$r6r7s rruleszTrackerPatternsDB.rulesNsM ****t}}*= *JCuQxq) ) *sA Ac #xKd}|jD]\}tjd| t|d}|j dk7r&tj d|j d|\n|tjdy|jd jD]Z}|d jd d |jd gDcgc]}|jd d c}|jdgf\y#t$r"}tj d||Yd}~d}~wwxYwcc}ww)Nz'TRACKER_PATTERNS: Trying to fetch %s...)timeoutz:TRACKER_PATTERNS: HTTPError (%s) occured while fetching %sz'TRACKER_PATTERNS: ClearURL ignore HTTP  z5TRACKER_PATTERNS: failed fetching ClearURL rule lists providers urlPatternz\\\ exceptionsr>) CLEAR_LIST_URLr r/http_getr warning status_codeerrorjsonvaluesreplacer )r$respurlexcr5s rr0z!TrackerPatternsDB.iter_clear_listSs2&& C II? E Q/ 3& EdFVFVEWWXY\X]^_   < IIM N IIK ,335 D\"**6486:hh|R6PQsVT*Q"%    XZ]_bc "Rs:(D: DBD:D5-D: D2D-'D:-D22D:rQc |}t|}|jD]}|j}|sntj||j j |sCd}||j jD]}tj||sd}n|rtt|j}|r|jD]o\} } ||j jD]N}tj|| stjd|j| | |j| | fPq|j!t#|}t%|}J||j jD]Y}tj||stjd|j||j!d}t%|}||k7r|Sy)zThe URL arguments are normalized and cleaned of tracker parameters. Returns bool ``True`` to use URL unchanged (``False`` to ignore URL). If URL should be modified, the returned string is the new URL to use. )rQFTz0TRACKER_PATTERNS: %s remove tracker arg: %s='%s')queryz-TRACKER_PATTERNS: %s remove tracker arg: '%s')rr>rTrematchr rrlistrcopyrr r/netlocremove_replacerr) r$rQnew_urlparsed_new_urlr5 query_str do_ignorepattern query_argsnamevals r clean_urlzTrackerPatternsDB.clean_urlos!g.JJL2 D+11I88D!7!78'BI 6 67 88GW- $I   04Y~?S?S5T0UJ!+!2;ID##' (<(<#=;88GT2II RTbTiTikoqt'--tSk: ;;"0!8!8y?T!8!U$^4 $DKK$8$89Gxx3 "QSaShShjst)7)@)@r)@)J",^"< [2 h c>NrN)rrrr3rHr r%r+r*RuleTyper:rr>r0strboolrerrrrrss'HN ! 4MM*x)* (!38ASATCZAr__main__)"__doc__typingr__all__rVcollections.abcr urllib.parserrrrhttpxr searx.data.corer r searx.networkr rI TYPE_CHECKING searx.cacher tuplergrXrffinalrrdbr>rprintrrrrysL   $CC*)??( d3ic* +VV Vr z  B XXZ ar