diff --git a/README.md b/README.md index f46eb9f..99b1e25 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Some features of morss: - Follow 301/meta redirects - Recover xml feeds with corrupt encoding - Supports gzip-compressed http content -- HTTP caching with different backends (in-memory/sqlite/mysql/redis/diskcache) +- HTTP caching with different backends (in-memory/redis/diskcache) - Works as server/cli tool - Deobfuscate various tracking links @@ -501,11 +501,6 @@ be dropped from the feed, even if they're cached. `-1` for unlimited. morss uses caching to make loading faster. There are 3 possible cache backends: - `(nothing/default)`: a simple python in-memory dict-like object. -- `CACHE=sqlite`: sqlite3 cache. Default file location is in-memory (i.e. it -will be cleared every time the program is run). Path can be defined with -`SQLITE_PATH`. -- `CACHE=mysql`: MySQL cache. Connection can be defined with the following -environment variables: `MYSQL_USER`, `MYSQL_PWD`, `MYSQL_DB`, `MYSQL_HOST` - `CACHE=redis`: Redis cache. Connection can be defined with the following environment variables: `REDIS_HOST`, `REDIS_PORT`, `REDIS_DB`, `REDIS_PWD` - `CACHE=diskcache`: disk-based cache. Target directory canbe defined with diff --git a/morss/caching.py b/morss/caching.py index cc867f6..b3950ef 100644 --- a/morss/caching.py +++ b/morss/caching.py @@ -16,7 +16,6 @@ # with this program. If not, see . import os -import pickle import threading import time from collections import OrderedDict @@ -51,83 +50,6 @@ class BaseCache: return True -try: - import sqlite3 # isort:skip -except ImportError: - pass - - -class SQLiteCache(BaseCache): - def __init__(self, path=':memory:'): - self.con = sqlite3.connect(path, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False) - - with self.con: - self.con.execute('CREATE TABLE IF NOT EXISTS data (ky UNICODE PRIMARY KEY, data BLOB, timestamp INT)') - self.con.execute('pragma journal_mode=WAL') - - self.trim() - - def __del__(self): - self.con.close() - - def trim(self): - with self.con: - self.con.execute('DELETE FROM data WHERE timestamp <= ( SELECT timestamp FROM ( SELECT timestamp FROM data ORDER BY timestamp DESC LIMIT 1 OFFSET ? ) foo )', (CACHE_SIZE,)) - - def __getitem__(self, key): - row = self.con.execute('SELECT * FROM data WHERE ky=?', (key,)).fetchone() - - if not row: - raise KeyError - - return row[1] - - def __setitem__(self, key, data): - with self.con: - self.con.execute('INSERT INTO data VALUES (?,?,?) ON CONFLICT(ky) DO UPDATE SET data=?, timestamp=?', (key, data, time.time(), data, time.time())) - - -try: - import pymysql.cursors # isort:skip -except ImportError: - pass - - -class MySQLCacheHandler(BaseCache): - def __init__(self, user, password, database, host='localhost'): - self.user = user - self.password = password - self.database = database - self.host = host - - with self.cursor() as cursor: - cursor.execute('CREATE TABLE IF NOT EXISTS data (ky VARCHAR(255) NOT NULL PRIMARY KEY, data MEDIUMBLOB, timestamp INT)') - - self.trim() - - def cursor(self): - return pymysql.connect(host=self.host, user=self.user, password=self.password, database=self.database, charset='utf8', autocommit=True).cursor() - - def trim(self): - with self.cursor() as cursor: - cursor.execute('DELETE FROM data WHERE timestamp <= ( SELECT timestamp FROM ( SELECT timestamp FROM data ORDER BY timestamp DESC LIMIT 1 OFFSET %s ) foo )', (CACHE_SIZE,)) - - def __getitem__(self, key): - cursor = self.cursor() - cursor.execute('SELECT * FROM data WHERE ky=%s', (key,)) - row = cursor.fetchone() - - if not row: - raise KeyError - - return row[1] - - def __setitem__(self, key, data): - with self.cursor() as cursor: - cursor.execute('INSERT INTO data VALUES (%s,%s,%s) ON DUPLICATE KEY UPDATE data=%s, timestamp=%s', - (key, data, time.time(), data, time.time())) - - class CappedDict(OrderedDict, BaseCache): def trim(self): if CACHE_SIZE >= 0: diff --git a/setup.py b/setup.py index 1c5370e..248afef 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( packages = [package_name], install_requires = ['lxml', 'bs4', 'python-dateutil', 'chardet'], extras_require = { - 'full': ['pymysql', 'redis', 'diskcache', 'gunicorn', 'setproctitle'], + 'full': ['redis', 'diskcache', 'gunicorn', 'setproctitle'], 'dev': ['pylint', 'pytest', 'pytest-cov'], }, python_requires = '>=2.7',