pictuga/morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: use BLOB in sqlite and drop "buffer"

Browse Source 
Can't really remember why "buffer" was introduced in the first place
This commit is contained in:
pictuga
2017-11-04 13:54:40 +01:00
parent 203ba10dbd
commit b7db78f631
1 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
morss/crawler.py
View File

@@ -22,7 +22,6 @@ try:
basestring
except NameError:
basestring = unicode = str
buffer = memoryview
MIMETYPE = {
@@ -337,7 +336,7 @@ class CacheHandler(BaseHandler):
return out
def save(self, url, code, msg, headers, data, timestamp):
self.cache[url] = (code, msg, unicode(headers), buffer(data), timestamp)
self.cache[url] = (code, msg, unicode(headers), data, timestamp)
def http_request(self, req):
(code, msg, headers, data, timestamp) = self.load(req.get_full_url())
@@ -481,7 +480,7 @@ class SQLiteCache:
self.con = sqlite3.connect(filename or sqlite_default, detect_types=sqlite3.PARSE_DECLTYPES, check_same_thread=False)
with self.con:
self.con.execute('CREATE TABLE IF NOT EXISTS data (url UNICODE PRIMARY KEY, code INT, msg UNICODE, headers UNICODE, data BYTES, timestamp INT)')
self.con.execute('CREATE TABLE IF NOT EXISTS data (url UNICODE PRIMARY KEY, code INT, msg UNICODE, headers UNICODE, data BLOB, timestamp INT)')
self.con.execute('pragma journal_mode=WAL')
def __del__(self):