From 36e2a1c3fdf7711ea8a149affc7a3234561977c4 Mon Sep 17 00:00:00 2001
From: pictuga <contact@pictuga.com>
Date: Tue, 12 May 2020 19:34:16 +0200
Subject: [PATCH] crawler: increase size limit from 100KiB to 500

I'm looking at you, worldbankgroup.csod.com/ats/careersite/search.aspx
---
 morss/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/morss/crawler.py b/morss/crawler.py
index 2ea52cb..509802b 100644
--- a/morss/crawler.py
+++ b/morss/crawler.py
@@ -93,7 +93,7 @@ def custom_handler(follow=None, delay=None, encoding=None):
     # & HTTPSHandler
 
     #handlers.append(DebugHandler())
-    handlers.append(SizeLimitHandler(100*1024)) # 100KiB
+    handlers.append(SizeLimitHandler(500*1024)) # 500KiB
     handlers.append(HTTPCookieProcessor())
     handlers.append(GZIPHandler())
     handlers.append(HTTPEquivHandler())