diff --git a/README.md b/README.md index 8418942..fd02098 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,16 @@ Simple REST-API for sending messages over the [Matrix](https://matrix.org) proto ## Features - Docker support -- Authentication support -- Sending encrypted messages +- Encryption support - UTF-8 support -- Markdown support for messages +- Markdown support - Tag support to embed emojis in the title by using their short codes -- Title support - Web UI to send messages -- Webhooks -- Pushing messages as json +- Webhook support +- Json support +- Title support +- Authentication support +- Automatic message purging ## Table of contents > Note: Matrix-Notifier probably contains a lot of bugs. diff --git a/bot/sync.py b/bot/sync.py index 7c01be3..252c5bf 100644 --- a/bot/sync.py +++ b/bot/sync.py @@ -23,17 +23,17 @@ async def save_id(con, cur, id): async def check(messages, client): - try: - data_dir = config.datadir_bot - if not os.path.exists(data_dir): - os.mkdir(data_dir) + data_dir = config.datadir_bot + if not os.path.exists(data_dir): + os.mkdir(data_dir) - con = sqlite3.connect(f"{data_dir}/ids.db") - cur = con.cursor() + con = sqlite3.connect(f"{data_dir}/ids.db") + cur = con.cursor() - cur.execute( - '''CREATE TABLE IF NOT EXISTS ids (id INT PRIMARY KEY)''') + cur.execute( + '''CREATE TABLE IF NOT EXISTS ids (id INT PRIMARY KEY)''') + try: cur.execute(f'''SELECT * FROM (SELECT * FROM ids ORDER BY id DESC LIMIT 250) sub ORDER BY id ASC''') data = cur.fetchall() ids = [] diff --git a/docker/example.env b/docker/example.env index 925c473..3147ca9 100644 --- a/docker/example.env +++ b/docker/example.env @@ -27,3 +27,8 @@ SERVER_HOSTNAME=server # Optional Settings: # SERVER_PORT=5505 + +# How long messages should be kept in hours +# message_preserve_time = 72 +# How often the purge job for clearing all messages should be run in minutes +# message_purge_interval = 30 diff --git a/docker/release-docker-compose.yml b/docker/release-docker-compose.yml index bf10e1c..5a3d543 100644 --- a/docker/release-docker-compose.yml +++ b/docker/release-docker-compose.yml @@ -3,7 +3,7 @@ version: "3.3" services: bot: container_name: "matrix-notifer-bot" - image: nlion/matrix-notifier-bot:v1.0.2 + image: nlion/matrix-notifier-bot:v1.2.0 depends_on: - server env_file: @@ -14,7 +14,7 @@ services: server: container_name: "matrix-notifer-server" - image: nlion/matrix-notifier-server:v1.0.2 + image: nlion/matrix-notifier-server:v1.2.0 ports: - "5505:5505" env_file: diff --git a/server/Dockerfile b/server/Dockerfile index 9d92e2c..7b663a5 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -15,6 +15,9 @@ ENV SERVER_PORT 5505 ENV authorization false ENV auth_secret "" +ENV message_preserve_time 72 +ENV message_purge_interval 30 + EXPOSE 5505 CMD [ "python3", "app.py" ] \ No newline at end of file diff --git a/server/app.py b/server/app.py index 8f58107..568e6dd 100644 --- a/server/app.py +++ b/server/app.py @@ -3,6 +3,7 @@ import os import json import logging +from apscheduler.schedulers.background import BackgroundScheduler import config import parser @@ -16,6 +17,7 @@ static_url_path='/static', template_folder='web/templates') +scheduler = BackgroundScheduler() @app.route("/", methods=['POST']) def post_messages(): @@ -34,11 +36,12 @@ def post_messages(): msg = parser.messageparse(parameter=parameter, body=body) - id = saver.save_to_db(msg) + id, message_time = saver.save_to_db(msg) - msg_json = json.dumps(dict([('Id', id), ('Channels', msg.channels), - ('Title', msg.title), ('Content', msg.content), - ('Tags', msg.tags), ('Markdown', msg.markdown)])) + msg_json = json.dumps(dict([('Id', id), ('time', message_time), + ('Channels', msg.channels), ('Title', msg.title), + ('Content', msg.content), ('Tags', msg.tags), + ('Markdown', msg.markdown)])) return msg_json, 200 @@ -59,11 +62,12 @@ def webhook_messages(): msg = parser.webhookmessageparse(parameter=parameter, content=message) - id = saver.save_to_db(msg) + id, message_time = saver.save_to_db(msg) - msg_json = json.dumps(dict([('Id', id), ('Channels', msg.channels), - ('Title', msg.title), ('Content', msg.content), - ('Tags', msg.tags), ('Markdown', msg.markdown)])) + msg_json = json.dumps(dict([('Id', id), ('time', message_time), + ('Channels', msg.channels), ('Title', msg.title), + ('Content', msg.content), ('Tags', msg.tags), + ('Markdown', msg.markdown)])) return msg_json, 200 @@ -86,11 +90,12 @@ def json_messages(): msg = parser.webhookmessageparse(parameter=parameter, content=message) - id = saver.save_to_db(msg) + id, message_time = saver.save_to_db(msg) - msg_json = json.dumps(dict([('Id', id), ('Channels', msg.channels), - ('Title', msg.title), ('Content', msg.content), - ('Tags', msg.tags), ('Markdown', msg.markdown)])) + msg_json = json.dumps(dict([('Id', id), ('time', message_time), + ('Channels', msg.channels), ('Title', msg.title), + ('Content', msg.content), ('Tags', msg.tags), + ('Markdown', msg.markdown)])) return msg_json, 200 @@ -114,7 +119,7 @@ def get_messages(): cur = con.cursor() cur.execute( - '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, channels text, title text, content text, tags text, markdown text)''') + '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, date text, channels text, title text, content text, tags text, markdown text)''') cur.execute( 'SELECT * FROM (SELECT * FROM messages ORDER BY id DESC LIMIT :limit) sub ORDER BY id ASC', {"limit": parameter.limit}) @@ -122,9 +127,10 @@ def get_messages(): data = cur.fetchall() message_data_list = [] for tuple in data: - message_data = dict([('Id', tuple[0]), ('Channels', json.loads(tuple[1])), - ('Title', tuple[2]), ('Content', tuple[3]), - ('Tags', json.loads(tuple[4])),('Markdown', tuple[5])]) + message_data = dict([('Id', tuple[0]), ('time', tuple[1]), + ('Channels', json.loads(tuple[2])), ('Title', tuple[3]), + ('Content', tuple[4]), ('Tags', json.loads(tuple[5])), + ('Markdown', tuple[6])]) message_data_list.append(message_data) content_data = json.dumps(message_data_list) @@ -153,7 +159,7 @@ def get_message_byid(message_id): cur = con.cursor() cur.execute( - '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, channels text, title text, content text, tags text, markdown text)''') + '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, date text, channels text, title text, content text, tags text, markdown text)''') ids = parser.id_parse(message_id) @@ -165,9 +171,10 @@ def get_message_byid(message_id): if not data == []: for tuple in data: - message_data = dict([('Id', tuple[0]), ('Channels', json.loads(tuple[1])), - ('Title', tuple[2]), ('Content', tuple[3]), - ('Tags', json.loads(tuple[4])), ('Markdown', tuple[5])]) + message_data = dict([('Id', tuple[0]), ('time', tuple[1]), + ('Channels', json.loads(tuple[2])), ('Title', tuple[3]), + ('Content', tuple[4]), ('Tags', json.loads(tuple[5])), + ('Markdown', tuple[6])]) message_data_list.append(message_data) if not message_data_list == []: @@ -186,4 +193,6 @@ def get_page(): if __name__ == "__main__": + job = scheduler.add_job(saver.clean_db, 'interval', minutes=config.message_purge_interval) + scheduler.start() app.run(host='0.0.0.0', port=config.port) diff --git a/server/config.py b/server/config.py index 291fc6a..536106a 100644 --- a/server/config.py +++ b/server/config.py @@ -14,6 +14,10 @@ # Cant contain any &'s or +'s auth_secret = "" +# How long the server should keep messages in hours +message_preserve_time = 72 +# How often the purge job should be run in minutes +message_purge_interval = 30 # Do Not Touch # If docker is used it will use the environment values @@ -23,6 +27,8 @@ datadir_server = "/data" authorization = os.environ.get('authorization', False) auth_secret = os.environ.get('auth_secret', False) + message_preserve_time = os.environ.get('message_preserve_time', False) + message_purge_interval = os.environ.get('message_purge_interval', False) if str(authorization).lower() == "true" or authorization == True: authorization = True @@ -31,3 +37,15 @@ quit(1) else: authorization = False +if not type(message_preserve_time) == type(1): + try: + message_preserve = int(message_preserve_time) + except ValueError: + logger.error("Wrong message_preserve format") + quit(1) +if not type(message_purge_interval) == type(1): + try: + message_purge_interval = int(message_purge_interval) + except ValueError: + logger.error("Wrong message_purge_interval format") + quit(1) diff --git a/server/requirements.txt b/server/requirements.txt index 69c8270..63ea45a 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1 +1,2 @@ -flask==2.2.3 \ No newline at end of file +flask==2.2.3 +apscheduler \ No newline at end of file diff --git a/server/saver.py b/server/saver.py index 3b5901b..8ac4a79 100644 --- a/server/saver.py +++ b/server/saver.py @@ -1,6 +1,7 @@ import sqlite3 import os import json +from datetime import datetime import config @@ -32,14 +33,34 @@ def save_to_db(msg): cur = con.cursor() cur.execute( - '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, channels text, title text, content text, tags text, markdown text)''') + '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, date text, channels text, title text, content text, tags text, markdown text)''') id = get_next_id(cur) channels = json.dumps(msg.channels) tags = json.dumps(msg.tags) + message_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') cur.execute( - 'INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)', (id, channels, msg.title, msg.content, tags, msg.markdown)) + 'INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?, ?)', (id, message_time, channels, msg.title, msg.content, tags, msg.markdown)) + + con.commit() + return id, message_time + + +def clean_db(): + message_preserve_time = config.message_preserve_time + + data_dir = config.datadir_server + if not os.path.exists(data_dir): + os.mkdir(data_dir) + + con = sqlite3.connect(f"{data_dir}/messages.db") + cur = con.cursor() + + cur.execute( + '''CREATE TABLE IF NOT EXISTS messages (id INT PRIMARY KEY, date text, channels text, title text, content text, tags text, markdown text)''') + + cur.execute( + f'DELETE FROM messages WHERE date < datetime("now", "localtime", "-{message_preserve_time} hours")') con.commit() - return id