pictuga/morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

crawler: 301 http code doesn't respect headers

Browse Source 
More or less according to the specs
This commit is contained in:
pictuga
2017-03-18 22:18:10 -10:00
parent f4abc4e8a4
commit 9ee6ff60e1
1 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
morss/crawler.py
View File

@@ -294,6 +294,12 @@ class BaseCacheHandler(BaseHandler):
# force refresh
return None
elif code == 301 and cache_age < 7*24*3600:
print('301 so cached')
# "301 Moved Permanently" has to be cached...as long as we want (awesome HTTP specs), let's say a week (why not?)
# use force_min=0 if you want to bypass this (needed for a proper refresh)
pass
elif self.force_min is None and ('no-cache' in cc_list
or 'no-store' in cc_list
or ('private' in cc_list and not self.private)):
@@ -308,11 +314,6 @@ class BaseCacheHandler(BaseHandler):
# still recent enough for us, use cache
pass
elif code == 301 and cache_age < 7*24*3600:
# "301 Moved Permanently" has to be cached...as long as we want (awesome HTTP specs), let's say a week (why not?)
# use force_min=0 if you want to bypass this (needed for a proper refresh)
pass
else:
# according to the www, we have to refresh when nothing is said
return None