pictuga/morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Cache http errors to save time.

Browse Source
This commit is contained in:
pictuga
2013-05-01 17:56:03 +02:00
parent 2f1ae1ce91
commit 5ee5dbf359
1 changed files with 19 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
morss.py
View File

@@ -199,7 +199,10 @@ class XMLMap(object):
return unicode(out) if self._str else out return unicode(out) if self._str else out
def __getitem__(self, tag): def __getitem__(self, tag):
return self.__getattr__(tag) if self.__contains__(tag):
return self.__getattr__(tag)
else:
return None
def __setattr__(self, tag, value): def __setattr__(self, tag, value):
if tag.startswith('_'): if tag.startswith('_'):
@@ -251,7 +254,6 @@ def EncDownload(url):
data = con.read() data = con.read()
except (urllib2.HTTPError, urllib2.URLError) as error: except (urllib2.HTTPError, urllib2.URLError) as error:
log(error) log(error)
log('http error')
return False return False
# meta-redirect # meta-redirect
@@ -303,17 +305,27 @@ def Fill(rss, cache):
item.link = "".join([(t[s[0]] if s[0] in t else "=") + s[1:] for s in url[1:]]) item.link = "".join([(t[s[0]] if s[0] in t else "=") + s[1:] for s in url[1:]])
log(item.link) log(item.link)
# check cache # check cache and previous errors
if item.link in cache: if item.link in cache:
log('cached') content = cache.get(item.link)
item.content = cache.get(item.link) if content == 'httperr':
return if cache.isYoungerThan(DELAY*60):
log('cached http err')
return
else:
log('old http error')
else:
log('cached')
item.content = cache.get(item.link)
return
# download # download
ddl = EncDownload(item.link) ddl = EncDownload(item.link)
if ddl is False: if ddl is False:
return item log('http error')
cache.set(item.link, 'httperr')
return
data, enc, url = ddl data, enc, url = ddl
log(enc) log(enc)