pictuga/morss
1
1
Fork 0
Code Issues Pull Requests Actions Packages Releases Activity

Detect encoding everytime

Browse Source
This commit is contained in:
pictuga
2020-04-07 10:38:36 +02:00
parent 7691df5257
commit f3d1f92b39
4 changed files with 19 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
morss/readabilite.py
View File

@@ -6,11 +6,14 @@ import re
def parse(data, encoding=None):
if encoding:
parser = lxml.html.HTMLParser(remove_blank_text=True, remove_comments=True, encoding=encoding)
else:
parser = lxml.html.HTMLParser(remove_blank_text=True, remove_comments=True)
data = BeautifulSoup(data, 'lxml', from_encoding=encoding).prettify('utf-8')
return lxml.html.fromstring(BeautifulSoup(data, 'lxml').prettify('utf-8'), parser=parser)
else:
data = BeautifulSoup(data, 'lxml').prettify('utf-8')
parser = lxml.html.HTMLParser(remove_blank_text=True, remove_comments=True, encoding='utf-8')
return lxml.html.fromstring(data, parser=parser)
def count_words(string):