Compare commits
3 Commits
dfb2b83c06
...
6529fdbdd8
Author | SHA1 | Date |
---|---|---|
pictuga | 6529fdbdd8 | |
pictuga | f4da40fffb | |
pictuga | d27fc93f75 |
|
@ -8,64 +8,59 @@ jobs:
|
|||
test-lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- run: apt-get -y update && apt-get -y install python3-pip libenchant-2-2 aspell-en
|
||||
- run: pip3 install .[full] .[dev]
|
||||
|
||||
- name: Prepare image
|
||||
run: apt-get -y update && apt-get -y install python3-pip libenchant-2-2 aspell-en
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip3 install .[full] .[dev]
|
||||
- run: isort --check-only --diff .
|
||||
- run: pylint morss --rcfile=.pylintrc --disable=C,R,W --fail-under=8
|
||||
- run: pytest --cov=morss tests
|
||||
|
||||
|
||||
publish-deploy:
|
||||
python-publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- run: apt-get -y update && apt-get -y install python3-pip python3-build
|
||||
- run: python3 -m build
|
||||
|
||||
- name: Prepare image
|
||||
run: apt-get -y update && apt-get -y install python3-pip python3-build
|
||||
|
||||
- name: Build package
|
||||
run: python3 -m build
|
||||
|
||||
- name: Publish package
|
||||
uses: https://github.com/pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
password: ${{ secrets.pypi_api_token }}
|
||||
- name: Push python package to the server
|
||||
uses: https://github.com/appleboy/scp-action@v0.1.4
|
||||
with:
|
||||
host: ${{ secrets.ssh_host }}
|
||||
username: ${{ secrets.ssh_user }}
|
||||
key: ${{ secrets.ssh_key }}
|
||||
source: dist/morss-*.tar.gz
|
||||
target: /home/ubuntu
|
||||
- name: Install & reload the server
|
||||
uses: https://github.com/appleboy/ssh-action@v0.1.10
|
||||
with:
|
||||
host: ${{ secrets.ssh_host }}
|
||||
username: ${{ secrets.ssh_user }}
|
||||
key: ${{ secrets.ssh_key }}
|
||||
script: |
|
||||
sudo pip install --upgrade dist/morss*-.tar.gz[full]
|
||||
sudo rm -r dist
|
||||
sudo morss-helper reload
|
||||
|
||||
docker-publish:
|
||||
docker-publish-deploy:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: catthehacker/ubuntu:act-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: https://github.com/docker/setup-qemu-action@v2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: https://github.com/docker/setup-buildx-action@v2
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: https://github.com/docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.docker_user }}
|
||||
password: ${{ secrets.docker_pwd }}
|
||||
|
||||
- name: Build and push
|
||||
uses: https://github.com/docker/build-push-action@v4
|
||||
with:
|
||||
|
@ -73,3 +68,11 @@ jobs:
|
|||
platforms: linux/amd64,linux/arm64,linux/arm/v7
|
||||
push: true
|
||||
tags: ${{ secrets.docker_repo }}
|
||||
|
||||
- name: Deploy on server
|
||||
uses: https://github.com/appleboy/ssh-action@v0.1.10
|
||||
with:
|
||||
host: ${{ secrets.ssh_host }}
|
||||
username: ${{ secrets.ssh_user }}
|
||||
key: ${{ secrets.ssh_key }}
|
||||
script: morss-update
|
||||
|
|
11
README.md
11
README.md
|
@ -81,9 +81,9 @@ From git
|
|||
pip install git+https://git.pictuga.com/pictuga/morss.git#egg=morss[full]
|
||||
```
|
||||
|
||||
The full install includes all the cache backends. Otherwise, only in-memory and
|
||||
sqlite3 caches are available. The full install also includes gunicorn (for more
|
||||
efficient HTTP handling).
|
||||
The full install includes all the cache backends. Otherwise, only in-memory
|
||||
cache is available. The full install also includes gunicorn (for more efficient
|
||||
HTTP handling).
|
||||
|
||||
The dependency `lxml` is fairly long to install (especially on Raspberry Pi, as
|
||||
C code needs to be compiled). If possible on your distribution, try installing
|
||||
|
@ -353,7 +353,7 @@ Using cache and passing arguments:
|
|||
```python
|
||||
>>> import morss
|
||||
>>> url = 'http://feeds.bbci.co.uk/news/rss.xml'
|
||||
>>> cache = '/tmp/morss-cache.db' # sqlite cache location
|
||||
>>> cache = '/tmp/morss-cache' # diskcache cache location
|
||||
>>> options = {'csv':True}
|
||||
>>> xml_string = morss.process(url, cache, options)
|
||||
>>> xml_string[:50]
|
||||
|
@ -367,11 +367,10 @@ under the hood.
|
|||
Doing it step-by-step:
|
||||
|
||||
```python
|
||||
import morss, morss.crawler
|
||||
import morss
|
||||
|
||||
url = 'http://newspaper.example/feed.xml'
|
||||
options = morss.Options(csv=True) # arguments
|
||||
morss.crawler.sqlite_default = '/tmp/morss-cache.db' # sqlite cache location
|
||||
|
||||
url, rss = morss.FeedFetch(url, options) # this only grabs the RSS feed
|
||||
rss = morss.FeedGather(rss, url, options) # this fills the feed and cleans it up
|
||||
|
|
|
@ -104,20 +104,7 @@ class DiskCacheHandler(BaseCache):
|
|||
|
||||
|
||||
if 'CACHE' in os.environ:
|
||||
if os.environ['CACHE'] == 'mysql':
|
||||
default_cache = MySQLCacheHandler(
|
||||
user = os.getenv('MYSQL_USER'),
|
||||
password = os.getenv('MYSQL_PWD'),
|
||||
database = os.getenv('MYSQL_DB'),
|
||||
host = os.getenv('MYSQL_HOST', 'localhost')
|
||||
)
|
||||
|
||||
elif os.environ['CACHE'] == 'sqlite':
|
||||
default_cache = SQLiteCache(
|
||||
os.getenv('SQLITE_PATH', ':memory:')
|
||||
)
|
||||
|
||||
elif os.environ['CACHE'] == 'redis':
|
||||
if os.environ['CACHE'] == 'redis':
|
||||
default_cache = RedisCacheHandler(
|
||||
host = os.getenv('REDIS_HOST', 'localhost'),
|
||||
port = int(os.getenv('REDIS_PORT', 6379)),
|
||||
|
|
|
@ -428,7 +428,7 @@ def process(url, cache=None, options=None):
|
|||
options = Options(options)
|
||||
|
||||
if cache:
|
||||
caching.default_cache = caching.SQLiteCache(cache)
|
||||
caching.default_cache = caching.DiskCacheHandler(cache)
|
||||
|
||||
url, rss = FeedFetch(url, options)
|
||||
rss = FeedGather(rss, url, options)
|
||||
|
|
Loading…
Reference in New Issue