From 8a4d68d72c98caf3dbed3fea26bc773859023119 Mon Sep 17 00:00:00 2001 From: pictuga Date: Sun, 5 Apr 2020 16:03:06 +0200 Subject: [PATCH] crawler: drop 'basic' toggle Can't even remember the use case --- morss/crawler.py | 6 ++---- morss/morss.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/morss/crawler.py b/morss/crawler.py index 456d5a0..78b8cd6 100644 --- a/morss/crawler.py +++ b/morss/crawler.py @@ -33,7 +33,7 @@ MIMETYPE = { DEFAULT_UA = 'Mozilla/5.0 (X11; Linux x86_64; rv:25.0) Gecko/20100101 Firefox/25.0' -def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=False): +def custom_handler(accept=None, strict=False, delay=None, encoding=None): handlers = [] # as per urllib2 source code, these Handelers are added first @@ -51,9 +51,7 @@ def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=F handlers.append(HTTPEquivHandler()) handlers.append(HTTPRefreshHandler()) handlers.append(UAHandler(DEFAULT_UA)) - - if not basic: - handlers.append(AutoRefererHandler()) + handlers.append(AutoRefererHandler()) handlers.append(EncodingFixHandler(encoding)) diff --git a/morss/morss.py b/morss/morss.py index de9fe24..4350a2a 100644 --- a/morss/morss.py +++ b/morss/morss.py @@ -336,7 +336,7 @@ def FeedFetch(url, options): try: con = crawler.custom_handler(accept='xml', strict=True, delay=delay, - encoding=options.encoding, basic=not options.items) \ + encoding=options.encoding) \ .open(url, timeout=TIMEOUT * 2) xml = con.read()