
    ibC                        U d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlmZ ddlmZmZ ded<   dZ ej*                  d	      Z ej,                         Zej1                         d
        Zej1                         ddedefd       ZddZej:                   G d dej<                               Zej@                   G d d             Z! G d dejD                        Z#ej:                   G d de#             Z$ej:                   G d dejJ                  e#             Z&ej:                   G d de#             Z'y)a  Implementations for caching favicons.

:py:obj:`FaviconCacheConfig`:
  Configuration of the favicon cache

:py:obj:`FaviconCache`:
  Abstract base class for the implementation of a favicon cache.

:py:obj:`FaviconCacheSQLite`:
  Favicon cache that manages the favicon BLOBs in a SQLite DB.

:py:obj:`FaviconCacheNull`:
  Fallback solution if the configured cache cannot be used for system reasons.

----

    N)sqlitedb)logger)humanize_byteshumanize_numberFaviconCacheCACHEs   FALLBACK_ICONzfavicons.cachec                  Z    t        t        j                         j                                y)zshow state of the cacheN)printr   statereport     %/root/searxng/searx/favicons/cache.pyr   r   -   s     
%++-


 !r   forcedebugc                 L   t        j                         }|r |j                  t         j                         nwg |_        t        j
                         }|j                  t        j                  d             t        j                  |       t        j                  t         j                         t        j                         }t        j                  |        t        j                         }||z
  }t        d       t        |j                  d      j                  d             y)z perform maintenance of the cachez%(message)s)r   zThe cache has been reduced by:z
- {descr}: {val}
N)logging	getLoggersetLevelDEBUGhandlersStreamHandlersetFormatter	Formatterr   
addHandlerr   r   maintenancer
   r   lstrip)r   r   root_loghandlerstate_t0state_t1state_deltas          r   r   r   3   s       "H'--('')W..}=>'"&{{}H	E"{{}HX%K	
*+	+

1
2
9
9$
?@r   FaviconCacheConfigc                 P   | j                   dk(  rOt        j                  dk  r0t        j                  dt        j
                         t        |       ayt        |       ay| j                   dk(  r!t        j                  d       t        |       ayt        d| j                    d      )	z$Initialization of a global ``CACHE``sqlite)   #   zUDisable favicon caching completely: SQLite library (%s) is too old! (require >= 3.35)memz<Favicons are cached in memory, don't use this in production!zfavicons db_type 'z' is unknownN)db_typesqlite3sqlite_version_infor   criticalsqlite_versionFaviconCacheNullr   FaviconCacheSQLiteerrorFaviconCacheMEMNotImplementedError)cfgs    r   initr5   H   s     {{h&&'1OOg&& %S)E&s+E		ST$!$6s{{m<"PQQr   c                       e Zd ZU dZdZej                  d   ed<   	  ej                         e
j                  z   dz   Zeed<   	 dZeed<   	 d	Zeed
<   	 dZeed<   	 dZeed<   	 dZej                  d   ed<   y)r$   z#Configuration of the favicon cache.r&   )r&   r)   r*   zfaviconcache.dbdb_urli ' 	HOLD_TIMEi   LIMIT_TOTAL_BYTESi P  BLOB_MAX_BYTESi  MAINTENANCE_PERIODauto)r<   offMAINTENANCE_MODEN)__name__
__module____qualname____doc__r*   tLiteral__annotations__tempfile
gettempdirossepr7   strr8   intr9   r:   r;   r>   r   r   r   r$   r$   \   s    -*2GQYY'2 &(%%'"&&03DDFCD>&Is&T-s-3 $NC#: &% 28aii.7	r   c                       e Zd ZU dZdZedz  ed<   dZedz  ed<   dZedz  ed<   dZ	edz  ed<   dde
fddefdd	e
fdd
effZeeeeej                   eegef   ez  f   df   ed<   ddZddefdZy)FaviconCacheStatsz@Dataclass which provides information on the status of the cache.Nfaviconsbytesdomains	resolversznumber of favicons in cachez#total size (approx. bytes) of cachez total number of domains in cacheznumber of resolvers.field_descrc                 D   t        || j                        s%t        d| j                   dt        |       d      i }| j                  D ]C  \  }}}t        | |      t        ||      }}d ||fv r&t        |t              r	||z
  ||<   ?|||<   E  | j                  di |S )Nz$unsupported operand type(s) for +: 'z' and ''r   )
isinstance	__class__	TypeErrortyperR   getattrrK   )selfotherkwargsfield_self_val	other_vals          r   __sub__zFaviconCacheStats.__sub__   s    %0B4>>BRRYZ^_dZeYffghii++ 	)KE1a")$"6u8MiH),,(C( (9 4u (u	) t~~'''r   fmtc                     g }| j                   D ]A  \  }}}t        | |      }|d}n ||      }|j                  |j                  ||             C dj	                  |      S )Nz--)descrval )rR   rY   appendformatjoin)rZ   rb   sr]   rd   castre   s          r   r   zFaviconCacheStats.report   sk    "&"2"2 	7E5$%dE2C{3iHHSZZeZ56	7 wwqzr   )r[   rM   returnrM   )z{descr}: {val}
)r?   r@   rA   rB   rN   rK   rE   rO   rP   rQ   r   r   rJ   rR   tuplerC   CallablerX   ra   r   r   r   r   rM   rM      s    JHcDjE3:GS4Z IsTz  
2OD	7H	6H	+S1	TKuS#qzz3*c/'BT'IIJCOP (	# 	r   rM   c                   $   e Zd ZdZej
                  defd       Zej
                  dededde	de
z  dez  f   z  fd       Zej
                  deded	edz  d
e
dz  def
d       Zej
                  defd       Zej
                  ddefd       Zy)r   z>Abstract base class for the implementation of a favicon cache.r4   c                      y)DAn instance of the favicon cache is build up from the configuration.Nr   rZ   r4   s     r   __init__zFaviconCache.__init__       r   resolver	authorityrl   Nc                      y)zReturns ``None`` or the tuple of ``(data, mime)`` that has been
        registered in the cache.  The ``None`` indicates that there was no entry
        in the cache.Nr   rZ   ru   rv   s      r   __call__zFaviconCache.__call__   rt   r   mimedatac                      y)zwSet data and mime-type in the cache.  If data is None, the
        :py:obj:`FALLBACK_ICON` is registered. in the cache.Nr   rZ   ru   rv   rz   r{   s        r   setzFaviconCache.set   rt   r   c                      y)zfReturns a :py:obj:`FaviconCacheStats` (key/values) with information
        on the state of the cache.Nr   rZ   s    r   r   zFaviconCache.state   rt   r   r   c                      y)z!Performs maintenance on the cacheNr   rZ   r   s     r   r   zFaviconCache.maintenance   rt   r   F)r?   r@   rA   rB   abcabstractmethodr$   rs   rJ   rm   rO   ry   boolr~   rM   r   r   r   r   r   r   r      s   HS. S S 	  dUlTX[^T^F^@_9_  
 	@C @C @sTz @QU @Z^ @ @ 	&( & & 	0 0 0r   c            
           e Zd ZdZdefdZdededdedez  dez  f   z  fdZ	deded	edz  d
edz  de
f
dZd Zdde
fdZy)r/   a*  A dummy favicon cache that caches nothing / a fallback solution. The
    NullCache is used when more efficient caches such as the
    :py:obj:`FaviconCacheSQLite` cannot be used because, for example, the SQLite
    library is only available in an old version and does not meet the
    requirements.r4   c                      y Nr   rr   s     r   rs   zFaviconCacheNull.__init__       r   ru   rv   rl   Nc                      y r   r   rx   s      r   ry   zFaviconCacheNull.__call__   r   r   rz   r{   c                      y)NFr   r}   s        r   r~   zFaviconCacheNull.set   s    r   c                     t        d      S )Nr   rN   )rM   r   s    r   r   zFaviconCacheNull.state   s     !,,r   r   c                      y r   r   r   s     r   r   zFaviconCacheNull.maintenance       r   r   r?   r@   rA   rB   r$   rs   rJ   rm   rO   ry   r   r~   r   r   r   r   r   r/   r/      s    .   dUlTX[^T^F^@_9_ C C sTz QU Z^ - r   r/   c            
           e Zd ZdZdZdZ	 dZ	 eedZdZ	 dZ	dZ
d	Zd
ef fdZdededdedez  dez  f   z  fdZdedededz  dedz  def
dZedefd       ZddefdZddedej2                  fdZdefdZ xZS )r0   a  Favicon cache that manages the favicon BLOBs in a SQLite DB.  The DB
    model in the SQLite DB is implemented using the abstract class
    :py:obj:`sqlitedb.SQLiteAppl`.

    For introspection of the DB, jump into developer environment and run command
    to show cache state::

        $ ./manage pyenv.cmd bash --norc --noprofile
        (py3) python -m searx.favicons cache state

    The following configurations are required / supported:

    - :py:obj:`FaviconCacheConfig.db_url`
    - :py:obj:`FaviconCacheConfig.HOLD_TIME`
    - :py:obj:`FaviconCacheConfig.LIMIT_TOTAL_BYTES`
    - :py:obj:`FaviconCacheConfig.BLOB_MAX_BYTES`
    - :py:obj:`MAINTENANCE_PERIOD`
    - :py:obj:`MAINTENANCE_MODE`
       zCREATE TABLE IF NOT EXISTS blobs (
  sha256     TEXT,
  bytes_c    INTEGER,
  mime       TEXT NOT NULL,
  data       BLOB NOT NULL,
  PRIMARY KEY (sha256))zCREATE TABLE IF NOT EXISTS blob_map (
    m_time     INTEGER DEFAULT (strftime('%s', 'now')),  -- last modified (unix epoch) time in sec.
    sha256     TEXT,
    resolver   TEXT,
    authority  TEXT,
    PRIMARY KEY (resolver, authority)))blobsblob_mapzDELETE FROM blobs WHERE sha256 IN ( SELECT b.sha256   FROM blobs b   LEFT JOIN blob_map bm     ON b.sha256 = bm.sha256  WHERE bm.sha256 IS NULL)zlSELECT b.sha256, b.bytes_c FROM blobs b  JOIN blob_map bm     ON b.sha256 = bm.sha256 ORDER BY bm.m_time ASCzfINSERT INTO blobs (sha256, bytes_c, mime, data) VALUES (?, ?, ?, ?)    ON CONFLICT (sha256) DO NOTHINGzINSERT INTO blob_map (sha256, resolver, authority) VALUES (?, ?, ?)    ON CONFLICT DO UPDATE    SET sha256=excluded.sha256, m_time=strftime('%s', 'now')r4   c                     |j                   dk(  rt        j                  d       t        |   |j                          || _        y)rq   z:memory:z/don't use SQLite DB in :memory: in production!!N)r7   r   r-   superrs   r4   )rZ   r4   rV   s     r   rs   zFaviconCacheSQLite.__init__8  s6     ::#OOMN$r   ru   rv   rl   Nc                    d}| j                   j                  |||f      j                         }|y d\  }}|d   }|t        k(  r||fS d}| j                   j                  ||f      j                         }||\  }}||fS )Nz@SELECT sha256 FROM blob_map WHERE resolver = ? AND authority = ?NNr   z-SELECT data, mime FROM blobs WHERE sha256 = ?)DBexecutefetchoneFALLBACK_ICON)rZ   ru   rv   sqlresr{   rz   sha256s           r   ry   zFaviconCacheSQLite.__call__@  s    PggoocHi#89BBD;!
dQ]":=ggoocF9-668?JD$Tzr   rz   r{   c           	         | j                   j                  dk(  r:t        t        j                               | j                  kD  r| j                          ||t        j                  d||       yt        |xs d      }|| j                   j                  kD  r t        j                  d|d|d|d       y|t        }n#t        j                  |      j                         }| j                         5 }|t        k7  r |j!                  | j"                  ||||f       |j!                  | j$                  |||f       d d d        j'                          y	# 1 sw Y   xY w)
Nr<   Bfavicon resolver %s tries to cache mime-type None for authority %sFr   zfavicon of resolver: z / authority: z to big to cache (bytes: z) T)r4   r>   rK   timenext_maintenance_timer   r   r1   lenr:   infor   hashlibr   	hexdigestconnectr   SQL_INSERT_BLOBSSQL_INSERT_BLOB_MAPclose)rZ   ru   rv   rz   r{   bytes_cr   conns           r   r~   zFaviconCacheSQLite.setR  s,   88$$.3tyy{3CdF`F`3`LLT
 dkc"TXX,,,KKZbdmovw <"F^^D)335F\\^ 	Rt&T22VWdD4QRLL11FHi3PQ	R 	

	R 	Rs   4A	EEc                 f    | j                   j                  | j                  j                  d      z   S )z2Returns (unix epoch) time of the next maintenance.LAST_MAINTENANCE)r4   r;   
propertiesm_timer   s    r   r   z(FaviconCacheSQLite.next_maintenance_timew  s)     xx**T__-C-CDV-WWWr   r   c                 ^   |s@t        t        j                               | j                  k  rt        j                  d       y | j
                  j                  dd       | j                         5 }|j                  d| j                  j                         }t        j                  d|j                         |j                  | j                        }t        j                  d|j                         |j                  d      j                         d   xs d}|| j                  j                  kD  r|| j                  j                  z
  }d}g }|j                  | j                        D ]$  }|\  }	}
|j!                  |	       ||
z  }||kD  s$ n |rf|j                  d	d
j#                  |      z         |j                  dd
j#                  |      z         t        j                  dt%        |      |       d d d        j                  d       |j'                          y # 1 sw Y   +xY w)NzGno maintenance required yet, next maintenance interval is in the futurer   rf   z^DELETE FROM blob_map WHERE cast(m_time as integer) < cast(strftime('%s', 'now') as integer) - z*dropped %s obsolete blob_map items from dbz!dropped %s obsolete BLOBS from dbSELECT SUM(bytes_c) FROM blobsr   z(DELETE FROM blobs WHERE sha256 IN ('%s')z','z+DELETE FROM blob_map WHERE sha256 IN ('%s')z,dropped %s blobs with total size of %s byteszPRAGMA wal_checkpoint(TRUNCATE))rK   r   r   r   r   r   r~   r   r   r4   r8   rowcountSQL_DROP_LEFTOVER_BLOBSr   r9   SQL_ITER_BLOBS_SHA256_BYTES_Crg   ri   r   r   )rZ   r   r   r   total_bytesxcsha_listrowr   r   s              r   r   zFaviconCacheSQLite.maintenance}  s   
 TYY[)D,F,FFLLbc.3
 \\^ 	ct ,,]]a]e]e]o]o\prC LLEs||T,,t;;<CLL<cllK ,,'GHQQSTUV[Z[KTXX777$(("<"<<&(<<(J(JK C&)OFGOOF+LA1u LL!KejjYaNb!bcLL!NQVQ[Q[\dQe!efLL!OQTU]Q^`ab7	c@ 	67

C	c 	cs   /DH#A+H##H,r   defaultc                 n    | j                   j                  |      j                         }||d   }||}|S )Nr   )r   r   r   )rZ   r   r   re   s       r   
_query_valzFaviconCacheSQLite._query_val  s:    ggooc"++-?a&C;C
r   c           	          t        | j                  dd      | j                  dd      | j                  dd      | j                  dd            S )NzSELECT count(*) FROM blobsr   r   zHSELECT count(*) FROM (SELECT authority FROM blob_map GROUP BY authority)zFSELECT count(*) FROM (SELECT resolver FROM blob_map GROUP BY resolver))rN   rO   rP   rQ   )rM   r   r   s    r   r   zFaviconCacheSQLite.state  sM     __%A1E//"BAFOO$npqroo&npqr	
 	
r   r   r   )r?   r@   rA   rB   	DB_SCHEMA	DDL_BLOBSDDL_BLOB_MAPDDL_CREATE_TABLESr   r   r   r   r$   rs   rJ   rm   rO   ry   r   r~   propertyrK   r   r   rC   Anyr   rM   r   __classcell__)rV   s   @r   r0   r0      s+   ( II C*L I  	%  D	" "	. 	F .   dUlTX[^T^F^@_9_ $#C #C #sTz #QU #Z^ #J Xs X X
. .`c AEE 
( 
r   r0   c            
           e Zd ZdZdefdZdededdeedz  edz  f   z  fdZ	deded	edz  d
edz  de
f
dZd Zdde
fdZy)r2   zFavicon cache in process' memory.  Its just a POC that stores the
    favicons in the memory of the process.

    .. attention::

       Don't use it in production, it will blow up your memory!!

    r4   c                 .    || _         i | _        i | _        y r   )r4   _data	_sha_mimerr   s     r   rs   zFaviconCacheMEM.__init__  s    ')
<>r   ru   rv   rl   Nc                     | j                   j                  | d| d      \  }}|y | j                  j                  |      }|t        k(  rd }||fS )N:r   )r   getr   r   )rZ   ru   rv   sharz   r{   s         r   ry   zFaviconCacheMEM.__call__  sX    NN&&(1YK'@,O	T;zz~~c"= DTzr   rz   r{   c                     |	t         }d }n|t        j                  d||       yt        j                  |      j                         }|| j                  |<   ||f| j                  | d| <   y)Nr   Fr   T)r   r   r1   r   r   r   r   r   )rZ   ru   rv   rz   r{   digests         r   r~   zFaviconCacheMEM.set  su    < DD\LLT
 %//1!

65;TN(1YK01r   c                 \    t        t        | j                  j                                     S )Nr   )rM   r   r   keysr   s    r   r   zFaviconCacheMEM.state  s     #djjoo.?*@AAr   r   c                      y r   r   r   s     r   r   zFaviconCacheMEM.maintenance  r   r   r   r   r   r   r   r2   r2     s    ?. ?  edlTWZ^T^F^@_9_ C C sTz QU Z^ &B r   r2   )TF)r4   r$   )(rB   typingrC   rH   r   dataclassesr   r   r+   rF   r   typermsgspecsearxr   r   searx.utilsr   r   rE   r   getChildTyperappcommandr   r   r   r5   finalStructr$   	dataclassrM   ABCr   r/   
SQLiteApplr0   r2   r   r   r   <module>r      sh  $  	 
           7  	)	*ekkm " "
 At A4 A A(R( - - 	-` & & &R0377 08 |  	. O
,,l O
 	O
d 1l 1 	1r   