diff --git a/morss/crawler.py b/morss/crawler.py index 2cafd4e..d5416eb 100644 --- a/morss/crawler.py +++ b/morss/crawler.py @@ -44,6 +44,7 @@ def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=F # FTPHandler, FileHandler, HTTPErrorProcessor] # & HTTPSHandler + #handlers.append(DebugHandler()) handlers.append(HTTPCookieProcessor()) handlers.append(GZIPHandler()) handlers.append(HTTPEquivHandler()) @@ -63,6 +64,21 @@ def custom_handler(accept=None, strict=False, delay=None, encoding=None, basic=F return build_opener(*handlers) +class DebugHandler(BaseHandler): + handler_order = 2000 + + def http_request(self, req): + print(repr(req.header_items())) + return req + + def http_response(self, req, resp): + print(resp.headers.__dict__) + return resp + + https_request = http_request + https_response = http_response + + class GZIPHandler(BaseHandler): def http_request(self, req): req.add_unredirected_header('Accept-Encoding', 'gzip')