From c2608165908886f15c523c58d760a61edb6d01f8 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 30 Jan 2026 21:13:49 +0100 Subject: [PATCH 01/25] More Supermarkets and Baecker --- settings/rule/00-default.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/settings/rule/00-default.json b/settings/rule/00-default.json index d4c51ee..32da5cc 100644 --- a/settings/rule/00-default.json +++ b/settings/rule/00-default.json @@ -39,7 +39,17 @@ "tags": ["Lebensmittel"], "filter": [{ "key": "text_tx", - "value": "(EDEKA|Wucherpfennig|Penny|Aldi|Kaufland|netto)", + "value": "((? Date: Fri, 30 Jan 2026 21:51:27 +0100 Subject: [PATCH 02/25] Load more now works with select all --- app/static/js/iban.js | 60 +++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/app/static/js/iban.js b/app/static/js/iban.js index b8ac3db..db0c424 100644 --- a/app/static/js/iban.js +++ b/app/static/js/iban.js @@ -1,34 +1,15 @@ "use strict"; +const selectAllCheckbox = document.getElementById('select-all'); let ROW_CHECKBOXES = null; let PAGE = 1; document.addEventListener('DOMContentLoaded', function () { // enabling/disabling the edit button based on checkbox selection - const selectAllCheckbox = document.getElementById('select-all'); ROW_CHECKBOXES = document.querySelectorAll('.row-checkbox'); - - selectAllCheckbox.addEventListener('change', function () { - ROW_CHECKBOXES.forEach(checkbox => { - checkbox.checked = selectAllCheckbox.checked; - }); - updateEditButtonState(); - listTxElements(); - }); - - ROW_CHECKBOXES.forEach(checkbox => { - checkbox.checked = false; - checkbox.addEventListener('change', function () { - if (!this.checked) { - selectAllCheckbox.checked = false; - } else if (Array.from(ROW_CHECKBOXES).every(cb => cb.checked)) { - selectAllCheckbox.checked = true; - } - updateEditButtonState(); - listTxElements(); - }); - }); + selectAllCheckbox.addEventListener('change', set_all_checkboxes); + ROW_CHECKBOXES.forEach(checkbox => set_row_checkboxes(checkbox)); // Tag Chip Bullets const inputTagContainers = [ @@ -68,6 +49,35 @@ document.addEventListener('DOMContentLoaded', function () { // -- DOM Functions ----------------------------------------------------------- // ---------------------------------------------------------------------------- +/** + * Set all checkboxes to the state of the headerbox + */ +function set_all_checkboxes() { + ROW_CHECKBOXES.forEach(checkbox => { + checkbox.checked = this.checked; + }); + updateEditButtonState(); + listTxElements(); +} + +/** + * Set an eventlistener for every box and change the header when unselected + * + * @param {DOMElement} checkbox + */ +function set_row_checkboxes(checkbox){ + checkbox.checked = false; + checkbox.addEventListener('change', function () { + if (!this.checked) { + selectAllCheckbox.checked = false; + } else if (Array.from(ROW_CHECKBOXES).every(cb => cb.checked)) { + selectAllCheckbox.checked = true; + } + updateEditButtonState(); + listTxElements(); + }); +} + /** * Clears information from a result Box * @@ -384,6 +394,12 @@ function loadMore() { // Append new Rows document.querySelector('.transactions tbody').innerHTML += responseText; + + // enabling/disabling the edit button based on checkbox selection + const selectAllCheckbox = document.getElementById('select-all'); + ROW_CHECKBOXES = document.querySelectorAll('.row-checkbox'); + selectAllCheckbox.addEventListener('change', set_all_checkboxes); + ROW_CHECKBOXES.forEach(checkbox => set_row_checkboxes(checkbox)); }); // Call URI From 57e357e6dfbaf4739a921aa18716dfab12181a61 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Mon, 2 Feb 2026 17:10:21 +0100 Subject: [PATCH 03/25] Fix Rule + Add Tagging Integ Tests --- settings/rule/00-default.json | 2 +- tests/test_integ_more_rules.py | 78 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/test_integ_more_rules.py diff --git a/settings/rule/00-default.json b/settings/rule/00-default.json index 32da5cc..8074973 100644 --- a/settings/rule/00-default.json +++ b/settings/rule/00-default.json @@ -39,7 +39,7 @@ "tags": ["Lebensmittel"], "filter": [{ "key": "text_tx", - "value": "((? Date: Mon, 2 Feb 2026 21:24:08 +0100 Subject: [PATCH 04/25] Take care when to merge lists in TinyDB update --- app/ui.py | 2 +- handler/TinyDb.py | 75 ++++++++++++++++++++--------------- tests/test_unit_handler_DB.py | 2 +- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/app/ui.py b/app/ui.py index 29c2ada..3fea933 100644 --- a/app/ui.py +++ b/app/ui.py @@ -280,7 +280,7 @@ def remove_tags(self, iban, t_id): 'compare': '==' }] - updated_entries = self.db_handler.update(new_data, iban, condition) + updated_entries = self.db_handler.update(new_data, iban, condition, merge=False) return updated_entries def remove_cat(self, iban, t_id): diff --git a/handler/TinyDb.py b/handler/TinyDb.py index e92be84..1540929 100644 --- a/handler/TinyDb.py +++ b/handler/TinyDb.py @@ -2,7 +2,6 @@ """Datenbankhandler für die Interaktion mit einer TinyDB Datenbankdatei.""" import os -import copy import operator import logging import re @@ -172,42 +171,56 @@ def update(self, data, collection, condition=None, multi='AND', merge=True): dict: - updated, int: Anzahl der aktualisierten Datensätze """ - # Form condition into a query and run - if condition is None: + # run update + docs_to_update = self.select(collection, condition, multi) + if not docs_to_update: + # No match, no update + return { 'updated': 0 } + + collection = self.connection.table(collection) + + # care about the right format + if data.get('tags') is not None and not isinstance(data.get('tags'), list): + data['tags'] = [data.get('tags')] + + # Result store for updated uuids + update_result = [] + + # Form condition into a query + if condition is None and merge: + logging.error('Using "merge" without a query is not possible') + return { 'error': 'Using "merge" without a query is not possible', 'updated': 0 } + + elif condition is None: query = Query().noop() else: query = self._form_complete_query(condition, multi) - # run update - new_tags = data.get('tags') - if new_tags and merge: - - docs_to_update = self.select(collection, condition, multi) - if not docs_to_update: - # No match, no update - return { 'updated': 0 } - - # Special handling with list to update (select - edit - store) - collection = self.connection.table(collection) - update_data = {} - update_result = [] - - # care about the right format - if not isinstance(new_tags, list): - data['tags'] = [new_tags] - - # Merge DB docs and update data and write it back to DB - for doc in docs_to_update: - existing_tags = doc.get('tags') or [] - update_data = copy.deepcopy(data) - update_data['tags'] = list(set(existing_tags + new_tags)) - update_result += collection.update(update_data, query) + if not merge: + # Update all at once (no merging) + update_result += collection.update(data, query) + return { 'updated': len(update_result) } - else: - # Plain function (overwrite existing attributes) - collection = self.connection.table(collection) - update_result = collection.update(data, query) + # Update every Entry one-by-one (merge every list item) + for doc in docs_to_update: + + # Look for lists to merge with this entry + for d in data.keys(): + + if isinstance(doc.get(d), list) and merge: + data[d] = list(set(doc.get(d) + data[d])) + continue + + # Update this uuid + update_result += collection.update( + data, + self._form_complete_query({ + 'key': 'uuid', + 'value': doc.get('uuid'), + 'compare': '==' + }) + ) return { 'updated': len(update_result) } diff --git a/tests/test_unit_handler_DB.py b/tests/test_unit_handler_DB.py index 44737bb..5a82779 100644 --- a/tests/test_unit_handler_DB.py +++ b/tests/test_unit_handler_DB.py @@ -262,7 +262,7 @@ def test_update(test_app): # Update all with one field data = {'art': 'Überweisung'} - updated_db = test_app.host.db_handler.update(data, 'DE89370400440532013000') + updated_db = test_app.host.db_handler.update(data, 'DE89370400440532013000', merge=False) update_all = updated_db.get('updated') assert update_all == 5, \ f'Es wurde nicht die richtige Anzahl geupdated (update_all): {update_all}' From 4c18b8a8a288b1a694e13d867432c744b58c8918 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 2 Feb 2026 21:38:21 +0100 Subject: [PATCH 05/25] Add MongoDB Tests to GitHub Action --- .github/workflows/python-app.yml | 69 +++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 319c3df..2aed73c 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -15,7 +15,7 @@ permissions: jobs: - PyTest: + PyTest_TinyDB: runs-on: ubuntu-latest @@ -49,7 +49,72 @@ jobs: all_tests=$(expr ${test_passed} + ${test_failed}) - echo "### Results" >> $GITHUB_STEP_SUMMARY + echo "### Results (with TinyDB)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| :checkered_flag: | :arrow_right_hook: | :x: |" >> $GITHUB_STEP_SUMMARY + echo "| ------------- | ------------- | ------------- |" >> $GITHUB_STEP_SUMMARY + echo "| $test_passed | $test_skipped | $test_failed |" >> $GITHUB_STEP_SUMMARY + + echo "$test_output" + if [[ $exitcode != 0 ]]; then + exit $exitcode + fi + exit 0 + + PyTest_MongoDB: + + runs-on: ubuntu-latest + + services: + mongodb: + image: mongo:8.2.4 + env: + MONGO_INITDB_ROOT_USERNAME: testuser + MONGO_INITDB_ROOT_PASSWORD: testpassword + ports: + - 27017:27017 + options: >- + --health-cmd mongo + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.12 + uses: actions/setup-python@v3 + with: + python-version: "3.12" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install -r requirements.txt + pip install -r tests/requirements.txt + - name: Adjust Config + run: | + sed -i -E s"|(DATABASE_BACKEND = )('tiny').*$|\1'mongo'|" tests/config.py + sed -i -E s"|(DATABASE_URI = )('/tmp').*$|\1'mongodb://testuser:testpassword@localhost:27017'|" tests/config.py + sed -i -E s"|(DATABASE_NAME = )('testdata.json').*$|\1'testdata'|" tests/config.py + - name: Test with pytest + run: | + set +e + + test_output="$(PYTHONPATH=. pytest -rN)" + exitcode=$? + + test_failed=$(sed -n '$s/^=*\s*\([0-9]*\)\sfailed.*/\1/p' <<< "$test_output" |tail -n1) + if [[ -z $test_failed ]]; then test_failed=0 ; fi + + test_passed=$(sed -n -E 's|^=*\s([0-9]+\sfailed,\s)?(([0-9]*)\spassed,\s)?.*|\3|p' <<< "$test_output" | tail -n1) + if [[ -z $test_passed ]]; then test_passed=0 ; fi + + test_skipped=$(tail -n1 <<< "$test_output" | rev |sed -E 's|^(.*deppiks\s([0-9]*))?.*|\2|') + if [[ -z $test_skipped ]]; then echo test_skipped=0 ; fi + + all_tests=$(expr ${test_passed} + ${test_failed}) + + echo "### Results (with MongoDB)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| :checkered_flag: | :arrow_right_hook: | :x: |" >> $GITHUB_STEP_SUMMARY echo "| ------------- | ------------- | ------------- |" >> $GITHUB_STEP_SUMMARY From dc7f21b4744f7bb94160aa78d3cfdcbf62758d8e Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 2 Feb 2026 22:00:21 +0100 Subject: [PATCH 06/25] remove newlines --- .github/workflows/python-app.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 2aed73c..0db91d5 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -2,23 +2,17 @@ # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python name: Python application - on: workflow_dispatch: push: branches: [ "master" ] pull_request: branches: [ "master" ] - permissions: contents: read - jobs: - PyTest_TinyDB: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 - name: Set up Python 3.12 @@ -60,11 +54,8 @@ jobs: exit $exitcode fi exit 0 - PyTest_MongoDB: - runs-on: ubuntu-latest - services: mongodb: image: mongo:8.2.4 @@ -78,7 +69,6 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - uses: actions/checkout@v3 - name: Set up Python 3.12 @@ -125,11 +115,8 @@ jobs: exit $exitcode fi exit 0 - PyLint: - runs-on: ubuntu-latest - steps: - name: Checkout repository uses: actions/checkout@v3 From b181e51f3661ca55dd3ab566447aac6afa28f44b Mon Sep 17 00:00:00 2001 From: Pitastic Date: Tue, 3 Feb 2026 12:20:08 +0100 Subject: [PATCH 07/25] Replace MongoDB health options with healthcheck --- .github/workflows/python-app.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 0db91d5..429e136 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -64,11 +64,12 @@ jobs: MONGO_INITDB_ROOT_PASSWORD: testpassword ports: - 27017:27017 - options: >- - --health-cmd mongo - --health-interval 10s - --health-timeout 5s - --health-retries 5 + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet + interval: 10s + timeout: 10s + retries: 5 + start_period: 40s steps: - uses: actions/checkout@v3 - name: Set up Python 3.12 From 01c0b158d375551dc1ae0aef6ff92e53a6ba51a3 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Tue, 3 Feb 2026 12:26:28 +0100 Subject: [PATCH 08/25] Refactor MongoDB healthcheck options in workflow Updated MongoDB healthcheck configuration in workflow. --- .github/workflows/python-app.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 429e136..726ebf1 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -64,12 +64,11 @@ jobs: MONGO_INITDB_ROOT_PASSWORD: testpassword ports: - 27017:27017 - healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet - interval: 10s - timeout: 10s - retries: 5 - start_period: 40s + options: >- + --health-cmd echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet + --health-interval 20s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v3 - name: Set up Python 3.12 From 4d7d96d1bf9e193597a828a99384d9405e7f6e93 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Tue, 3 Feb 2026 14:14:20 +0100 Subject: [PATCH 09/25] Fix health command syntax in MongoDB service --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 726ebf1..f0f5149 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -65,7 +65,7 @@ jobs: ports: - 27017:27017 options: >- - --health-cmd echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet + --health-cmd "echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet" --health-interval 20s --health-timeout 5s --health-retries 5 From 6654c2e896f703317ee9b7a6b1ea3d5c4be44cce Mon Sep 17 00:00:00 2001 From: Pitastic Date: Tue, 3 Feb 2026 14:22:33 +0100 Subject: [PATCH 10/25] modify sed expressions Updated database configuration settings in tests/config.py for testing. --- .github/workflows/python-app.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index f0f5149..4fec879 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -81,15 +81,14 @@ jobs: pip install pytest pip install -r requirements.txt pip install -r tests/requirements.txt - - name: Adjust Config - run: | - sed -i -E s"|(DATABASE_BACKEND = )('tiny').*$|\1'mongo'|" tests/config.py - sed -i -E s"|(DATABASE_URI = )('/tmp').*$|\1'mongodb://testuser:testpassword@localhost:27017'|" tests/config.py - sed -i -E s"|(DATABASE_NAME = )('testdata.json').*$|\1'testdata'|" tests/config.py - name: Test with pytest run: | set +e + sed -i -E s"|(DATABASE_BACKEND = )('tiny').*$|\1'mongo'|m" tests/config.py + sed -i -E s"|(DATABASE_URI = )('/tmp.*').*$|\1'mongodb://testuser:testpassword@localhost:27017'|" tests/config.py + sed -i -E s"|(DATABASE_NAME = )('testdata.json').*$|\1'testdata'|m" tests/config.py + test_output="$(PYTHONPATH=. pytest -rN)" exitcode=$? From 723c6b8313633fe6a9c3367230ad33b40f97f9b9 Mon Sep 17 00:00:00 2001 From: = <=> Date: Tue, 3 Feb 2026 16:06:13 +0100 Subject: [PATCH 11/25] Implement default testing for custom data (Step 1) --- settings/rule/00-default-categories.json | 26 ++++++++ .../{00-default.json => 00-default-tags.json} | 36 ++++------- tests/conftest.py | 2 +- tests/test_integ_external-db_rules.py | 64 +++++++++++++++++++ 4 files changed, 102 insertions(+), 26 deletions(-) create mode 100644 settings/rule/00-default-categories.json rename settings/rule/{00-default.json => 00-default-tags.json} (57%) create mode 100644 tests/test_integ_external-db_rules.py diff --git a/settings/rule/00-default-categories.json b/settings/rule/00-default-categories.json new file mode 100644 index 0000000..c26b309 --- /dev/null +++ b/settings/rule/00-default-categories.json @@ -0,0 +1,26 @@ +[ + { + "metatype": "category", + "name": "Lebensmittel", + "category": "Lebensmittel", + "filter": [{ + "key": "tags", + "value": ["Supermarkt", "Drogerie", "Apotheke", "Bäcker"], + "compare": "in" + }] + }, + { + "metatype": "category", + "name": "Abgaben", + "category": "Öffentliche Ausgaben", + "multi": "OR", + "filter": [{ + "key": "tags", + "value": ["Stadt", "Steuer"], + "compare": "in" + }], + "parsed": { + "Gläubiger-ID": "DE7000100000077777" + } + } +] \ No newline at end of file diff --git a/settings/rule/00-default.json b/settings/rule/00-default-tags.json similarity index 57% rename from settings/rule/00-default.json rename to settings/rule/00-default-tags.json index 8074973..965f2d0 100644 --- a/settings/rule/00-default.json +++ b/settings/rule/00-default-tags.json @@ -1,28 +1,4 @@ [ - { - "metatype": "category", - "name": "Lebensmittel", - "category": "Lebensmittel und Haushalt", - "filter": [{ - "key": "tags", - "value": ["Supermarkt", "Drogerie", "Apotheke"], - "compare": "in" - }] - }, - { - "metatype": "category", - "name": "Abgaben", - "category": "Öffentliche Ausgaben", - "multi": "OR", - "filter": [{ - "key": "tags", - "value": ["Stadt", "Steuer"], - "compare": "in" - }], - "parsed": { - "Gläubiger-ID": "DE7000100000077777" - } - }, { "metatype": "rule", "name": "City Tax", @@ -49,7 +25,17 @@ "tags": ["Bäcker"], "filter": [{ "key": "text_tx", - "value": "(SCHAEFERS\\sBY\\sEDEKA|MEISTERBAECKEREI|BAECKER\\sGOEING|Bäckerei|Baeckerei )", + "value": "(SCHAEFERS\\sBY\\sEDEKA|MEISTERBAECKEREI|BAECKER\\sGOEING|Bäckerei|Baeckerei)", + "compare": "regex" + }] + }, + { + "metatype": "rule", + "name": "OnlinePayment", + "tags": ["online"], + "filter": [{ + "key": "text_tx", + "value": "()", "compare": "regex" }] } diff --git a/tests/conftest.py b/tests/conftest.py index 90822ae..fb1e197 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,7 +33,7 @@ def test_app(): with app.app_context(): yield app - shutil.rmtree("/tmp/pynance-test", ignore_errors=True) + #shutil.rmtree("/tmp/pynance-test", ignore_errors=True) @pytest.fixture(scope="module") def mocked_db(): diff --git a/tests/test_integ_external-db_rules.py b/tests/test_integ_external-db_rules.py new file mode 100644 index 0000000..9957401 --- /dev/null +++ b/tests/test_integ_external-db_rules.py @@ -0,0 +1,64 @@ +#!/usr/bin/python3 # pylint: disable=invalid-name +""" +Testing rules on optional data (own data) not from this repo. + +Put (multiple) testfiles in /tmp/pynance-*.csv which will be imported with the Generic importer. +(See tests/input_generic.csv for an example file format) + +Fill in the result dict which tags or categories should be found for which uuid. +You need to know the tx_ids/uuids beforehand. + +The 'uuid' from the dict and from the will then be used to check, if the expected results match with this transaction. + +This Test will also pass if no testfiles are found. +""" + +import os +import sys +import glob + +# Add Parent for importing from 'app.py' +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(parent_dir) + +from helper import check_transaktion_list +from reader.Generic import Reader as Generic + + +EXPECTED_RESULTS = { + '786e1d4e16832aa321a0176c854fe087': { + 'tags': ["Stadt", "Tag2"], + 'category': '' + }, +} + +def test_rules_with_custom_input(test_app): + + for csv_file in glob.glob(os.path.join('/tmp', 'pynance-*.csv')): + + with test_app.app_context(): + transaction_list = Generic().from_csv(csv_file) + assert transaction_list, f"No transactions found in CSV file {csv_file}" + + # Check Reader Ergebnisse + check_transaktion_list(transaction_list) + print(10*'+', transaction_list) + + # Find the transaction by its ID + for tx in transaction_list: + tx_id = tx.get('uuid') + print(f"Checking tx_id {tx_id}...") + + if tx_id not in EXPECTED_RESULTS: + print(f"Skipping tx_id {tx_id} as it is not in EXPECTED_RESULTS") + continue + + # Check the right tags + assert len(EXPECTED_RESULTS[tx_id]['tags']) == len(tx.get('tags', [])), \ + f"Tags length mismatch for tx_id {tx_id}" + assert set(EXPECTED_RESULTS[tx_id]['tags']) == set(tx.get('tags', [])), \ + f"Tags mismatch for tx_id {tx_id}" + + # Check the right category + assert EXPECTED_RESULTS[tx_id]['category'] == tx.get('category'), \ + f"Category mismatch for tx_id {tx_id}" From c6848914538fd29f8f90a2ad5d5e964dc8d272cd Mon Sep 17 00:00:00 2001 From: Pitastic Date: Tue, 3 Feb 2026 16:34:14 +0100 Subject: [PATCH 12/25] Generic Test for external data implemented --- settings/rule/00-default-tags.json | 2 +- tests/test_integ_external-db_rules.py | 32 ++++++++++++++++++--------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/settings/rule/00-default-tags.json b/settings/rule/00-default-tags.json index 965f2d0..5f9b0f6 100644 --- a/settings/rule/00-default-tags.json +++ b/settings/rule/00-default-tags.json @@ -35,7 +35,7 @@ "tags": ["online"], "filter": [{ "key": "text_tx", - "value": "()", + "value": "(Ayden)", "compare": "regex" }] } diff --git a/tests/test_integ_external-db_rules.py b/tests/test_integ_external-db_rules.py index 9957401..b19856d 100644 --- a/tests/test_integ_external-db_rules.py +++ b/tests/test_integ_external-db_rules.py @@ -2,13 +2,17 @@ """ Testing rules on optional data (own data) not from this repo. -Put (multiple) testfiles in /tmp/pynance-*.csv which will be imported with the Generic importer. -(See tests/input_generic.csv for an example file format) +Put (multiple) testfiles in /tmp/pynance-*.csv which will be imported with +the Generic importer (see tests/input_generic.csv for an example file format). Fill in the result dict which tags or categories should be found for which uuid. You need to know the tx_ids/uuids beforehand. -The 'uuid' from the dict and from the will then be used to check, if the expected results match with this transaction. +The 'uuid' from the dict and from the will then be used to check, +if the expected results match with this transaction. + +The example in this code shows Transaction with uuid '524a0184ca2ba4a5e438f362da95cffc' +from the 'tests/input_generic.csv'' file. This Test will also pass if no testfiles are found. """ @@ -22,32 +26,38 @@ sys.path.append(parent_dir) from helper import check_transaktion_list -from reader.Generic import Reader as Generic EXPECTED_RESULTS = { - '786e1d4e16832aa321a0176c854fe087': { - 'tags': ["Stadt", "Tag2"], - 'category': '' + '524a0184ca2ba4a5e438f362da95cffc': { + 'tags': ["Lebensmittel"], + 'category': None }, } def test_rules_with_custom_input(test_app): + """Generic Tester for all transactions in CSV files as described above""" for csv_file in glob.glob(os.path.join('/tmp', 'pynance-*.csv')): with test_app.app_context(): - transaction_list = Generic().from_csv(csv_file) - assert transaction_list, f"No transactions found in CSV file {csv_file}" + fake_iban = 'DE89370400440532013000' + transaction_list = test_app.host.read_input(csv_file, bank='Generic', data_format='csv') # Check Reader Ergebnisse + assert transaction_list, f"No transactions found in CSV file {csv_file}" check_transaktion_list(transaction_list) - print(10*'+', transaction_list) + assert test_app.host.db_handler.insert(transaction_list, fake_iban), \ + "Inserting transactions from CSV failed" + assert test_app.host.tagger.tag_and_cat(fake_iban), \ + "Tagging and Categorizing transactions from CSV failed" + + # Re-Select all transactions for this IBAN + transaction_list = test_app.host.db_handler.select(fake_iban) # Find the transaction by its ID for tx in transaction_list: tx_id = tx.get('uuid') - print(f"Checking tx_id {tx_id}...") if tx_id not in EXPECTED_RESULTS: print(f"Skipping tx_id {tx_id} as it is not in EXPECTED_RESULTS") From e133e49566ed23a35035cde5f232381c84f989f5 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Wed, 4 Feb 2026 10:18:59 +0100 Subject: [PATCH 13/25] Weitere Regeln teilw. vorbereitet --- settings/rule/00-default-categories.json | 26 +++++++++++++++++++++++- settings/rule/00-default-tags.json | 14 +++++++++++-- tests/test_integ_external-db_rules.py | 3 ++- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/settings/rule/00-default-categories.json b/settings/rule/00-default-categories.json index c26b309..f26f5bc 100644 --- a/settings/rule/00-default-categories.json +++ b/settings/rule/00-default-categories.json @@ -12,15 +12,39 @@ { "metatype": "category", "name": "Abgaben", - "category": "Öffentliche Ausgaben", + "category": "Abgaben", "multi": "OR", "filter": [{ "key": "tags", "value": ["Stadt", "Steuer"], "compare": "in" + },{ + "key": "tags", + "value": ["Betreuung", "Bildung", "Verkehr"], + "compare": "notin" }], "parsed": { "Gläubiger-ID": "DE7000100000077777" } + }, + { + "metatype": "category", + "name": "Fixkosten", + "category": "Fixkosten", + "filter": [{ + "key": "tags", + "value": ["Strom", "Wasser", "Gas", "Versicherung", "Kommunikation", "Miete", "Kredite"], + "compare": "in" + }] + }, + { + "metatype": "category", + "name": "Fortbewegung", + "category": "Fortbewegung", + "filter": [{ + "key": "tags", + "value": ["Tanken", "Auto", "ÖPNV"], + "compare": "in" + }] } ] \ No newline at end of file diff --git a/settings/rule/00-default-tags.json b/settings/rule/00-default-tags.json index 5f9b0f6..274a8ea 100644 --- a/settings/rule/00-default-tags.json +++ b/settings/rule/00-default-tags.json @@ -15,7 +15,7 @@ "tags": ["Lebensmittel"], "filter": [{ "key": "text_tx", - "value": "((? Date: Wed, 4 Feb 2026 10:22:55 +0100 Subject: [PATCH 14/25] Fix rule with multiple filters --- settings/rule/00-default-categories.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings/rule/00-default-categories.json b/settings/rule/00-default-categories.json index f26f5bc..86efd0b 100644 --- a/settings/rule/00-default-categories.json +++ b/settings/rule/00-default-categories.json @@ -13,7 +13,7 @@ "metatype": "category", "name": "Abgaben", "category": "Abgaben", - "multi": "OR", + "multi": "AND", "filter": [{ "key": "tags", "value": ["Stadt", "Steuer"], From fb2089d8a426151c1dabb6311ad5fae2acc1dd05 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Wed, 4 Feb 2026 20:40:12 +0100 Subject: [PATCH 15/25] Add Src Konto to Details --- app/templates/iban.html | 4 ++++ app/templates/tx.html | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/app/templates/iban.html b/app/templates/iban.html index 3ed06ea..5eebf4a 100644 --- a/app/templates/iban.html +++ b/app/templates/iban.html @@ -218,6 +218,10 @@

Transaktions Details

Art: + + Konto: + + Gegenkonto: diff --git a/app/templates/tx.html b/app/templates/tx.html index 7cc37c2..8068123 100644 --- a/app/templates/tx.html +++ b/app/templates/tx.html @@ -35,10 +35,6 @@

- - Art: - {{tx.art}} - Datum (Wertstellnug) {{ tx.valuta | ctime }} @@ -47,9 +43,21 @@

Datum (Transaktion) {{ tx.date_tx | ctime }} + + Art: + {{tx.art}} + + + Konto: + {{tx.iban}} + + + Gegenkonto: + {{tx.gegenkonto}} + Betrag - {{tx.amount}} {{tx.currency}} + {{ "%.2f"|format(tx.amount|round(2)) }} {{tx.currency}} Buchungstext From 4418d9cc2524b0254a658f14a69376759e24e407 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Wed, 4 Feb 2026 21:00:53 +0100 Subject: [PATCH 16/25] Add Kredit-rule, fix custom rule in ui --- app/routes.py | 2 +- settings/rule/00-default-tags.json | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/app/routes.py b/app/routes.py index e501754..cfd8fad 100644 --- a/app/routes.py +++ b/app/routes.py @@ -553,7 +553,7 @@ def tag_and_cat(iban) -> dict: iban, category=custom_rule.get('category'), tags=custom_rule.get('tags'), - filters=custom_rule.get('filters'), + filters=custom_rule.get('filter'), parsed_keys=list(custom_rule.get('parsed', {}).keys()), parsed_vals=list(custom_rule.get('parsed', {}).values()), multi=custom_rule.get('multi', 'AND'), diff --git a/settings/rule/00-default-tags.json b/settings/rule/00-default-tags.json index 274a8ea..2be4429 100644 --- a/settings/rule/00-default-tags.json +++ b/settings/rule/00-default-tags.json @@ -15,7 +15,7 @@ "tags": ["Lebensmittel"], "filter": [{ "key": "text_tx", - "value": "((? Date: Wed, 4 Feb 2026 22:28:08 +0100 Subject: [PATCH 17/25] Fix Tagging in Groups (TinyDB) --- .gitignore | 2 +- app/routes.py | 1 - handler/TinyDb.py | 14 +++++++++++--- tests/conftest.py | 2 +- tests/test_integ_basics.py | 39 ++++++++++++++++++++++++++++++++++++-- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 656328a..6128e7a 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,4 @@ cython_debug/ # End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,python # Credentials: -config.conf \ No newline at end of file +settings/*/0[^0]*.json diff --git a/app/routes.py b/app/routes.py index cfd8fad..c21ada4 100644 --- a/app/routes.py +++ b/app/routes.py @@ -446,7 +446,6 @@ def uploadRules(metadata): Returns: json: Informationen zur Datei und Ergebnis der Untersuchung. """ - print(request.files) input_file = request.files.get('settings-input') if not input_file: return {'error': 'Es wurde keine Datei übermittelt.'}, 400 diff --git a/handler/TinyDb.py b/handler/TinyDb.py index 1540929..17c332a 100644 --- a/handler/TinyDb.py +++ b/handler/TinyDb.py @@ -177,8 +177,6 @@ def update(self, data, collection, condition=None, multi='AND', merge=True): # No match, no update return { 'updated': 0 } - collection = self.connection.table(collection) - # care about the right format if data.get('tags') is not None and not isinstance(data.get('tags'), list): data['tags'] = [data.get('tags')] @@ -197,14 +195,24 @@ def update(self, data, collection, condition=None, multi='AND', merge=True): else: query = self._form_complete_query(condition, multi) - if not merge: + if not merge and self.check_collection_is_iban(collection): # Update all at once (no merging) + collection = self.connection.table(collection) update_result += collection.update(data, query) return { 'updated': len(update_result) } + if not merge and not self.check_collection_is_iban(collection): + # Update all at once (no merging) but loop ibans in group + for c in self.get_group_ibans(collection): + collection = self.connection.table(c) + update_result += collection.update(data, query) + return { 'updated': len(update_result) } + # Update every Entry one-by-one (merge every list item) for doc in docs_to_update: + collection = self.connection.table(doc.get('iban')) + # Look for lists to merge with this entry for d in data.keys(): diff --git a/tests/conftest.py b/tests/conftest.py index fb1e197..90822ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,7 +33,7 @@ def test_app(): with app.app_context(): yield app - #shutil.rmtree("/tmp/pynance-test", ignore_errors=True) + shutil.rmtree("/tmp/pynance-test", ignore_errors=True) @pytest.fixture(scope="module") def mocked_db(): diff --git a/tests/test_integ_basics.py b/tests/test_integ_basics.py index 0d913c0..4b24cdc 100644 --- a/tests/test_integ_basics.py +++ b/tests/test_integ_basics.py @@ -270,6 +270,28 @@ def test_get_tx(test_app): "Der Inhalt der Testtransaktion wurde nicht wie erwartet selektiert" +def test_add_and_get_group(test_app): + """ + Testet das Hinzufügen einer Gruppe in der Instanz. + """ + with test_app.app_context(): + + result = test_app.host.db_handler.add_iban_group("testgroup", ["DE89370400440532013000"]) + assert result == {'inserted': 1}, 'Die Gruppe wurde nicht hinzugefügt.' + + # No Doublettes + result = test_app.host.db_handler.add_iban_group("testgroup", + ["DE89370400440532013000", "DE89370400440532011111"]) + assert result == {'inserted': 1}, \ + 'Die Gruppe wurde nicht geupdated.' + + result = test_app.host.db_handler.get_group_ibans("testgroup") + assert isinstance(result, list), 'Die IBANs wurden nicht als Liste zurückgegeben.' + assert len(result) == 2, 'Die Gruppe enthält nicht die erwartete Anzahl an IBANs.' + assert "DE89370400440532013000" in result, 'Die erste IBAN wurde nicht zurückgegeben.' + assert 'DE89370400440532011111' in result, 'Die zweite IBAN wurde nicht zurückgegeben.' + + def test_tag_stored_rules(test_app): """Testet das Tagging über den API Endpunkt: - Tagging mit einer definierten Regel @@ -375,8 +397,8 @@ def test_tag_custom_rules(test_app): 'rule_name': 'ui_selected_custom', 'rule': { 'tags': ['Supermarkt'], - 'filters': [ - {'key':'text_tx', 'value': r'EDEKA', 'compare': 'regex'} + 'filter': [ + {'key':'text_tx', 'value': 'EDEKA', 'compare': 'regex'} ] } } @@ -531,6 +553,19 @@ def test_tag_manual(test_app): assert tags == ['Replaced_TAG'], \ "Es wurden falsche Tags gespeichert" + # Check Tagging within a Group + new_tag = { + 'tags': ['Tagged in Group'], + 'overwrite': True + } + r = client.put( + "/api/setManualTag/testgroup/786e1d4e16832aa321a0176c854fe087", + json=new_tag + ) + r = r.json + assert r.get('updated') == 1, \ + "Der Eintrag (in der Gruppe) wurde nicht erneut aktualisiert" + def test_categorize_manual(test_app): """Testet das Kategorisieren über den API Endpunkt: From b043433f1620ce3c82e02665d51495b50d247fd6 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Wed, 4 Feb 2026 22:44:17 +0100 Subject: [PATCH 18/25] Solve Groups for delete and update in BaseDB --- handler/BaseDb.py | 46 +++++++++++++++++++++++++++++++++++++++++++--- handler/MongoDb.py | 4 ++-- handler/TinyDb.py | 17 +++++------------ 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/handler/BaseDb.py b/handler/BaseDb.py index 45c57bf..4ab0b02 100644 --- a/handler/BaseDb.py +++ b/handler/BaseDb.py @@ -160,8 +160,8 @@ def _insert(self, data: dict|list[dict], collection: str): """ raise NotImplementedError() - def update(self, data: dict, collection: str, condition: dict|list[dict], - multi:str, merge:bool=True): + def update(self, data: dict, collection: str, condition: dict|list[dict]=None, + multi:str='AND', merge:bool=True): """ Aktualisiert Datensätze in der Datenbank, die die angegebene Bedingung erfüllen. @@ -184,9 +184,29 @@ def update(self, data: dict, collection: str, condition: dict|list[dict], dict: - updated, int: Anzahl der aktualisierten Datensätze """ + if self.check_collection_is_iban(collection): + # Directly update IBAN collection + return self._update(data, collection, condition, multi, merge) + + # Update all IBANs in group + update_result = 0 + for iban in self.get_group_ibans(collection): + update_result += self._update(data, iban, condition, multi, merge).get('updated', 0) + + return {'updated': update_result} + + def _update(self, data: dict, collection: str, condition: dict|list[dict], + multi:str, merge:bool=True): + """ + Private Methode zum Aktualisieren von Datensätzen in der Datenbank, + Siehe 'update' Methode. + Returns: + dict: + - updated, int: Anzahl der aktualisierten Datensätze + """ raise NotImplementedError() - def delete(self, collection: str, condition: dict | list[dict]): + def delete(self, collection: str, condition: dict | list[dict]=None, multi: str='AND'): """ Löscht Datensätze in der Datenbank, die die angegebene Bedingung erfüllen. @@ -201,6 +221,26 @@ def delete(self, collection: str, condition: dict | list[dict]): - 'regex' : value wird als RegEx behandelt multi (str) : ['AND' | 'OR'] Wenn 'condition' eine Liste mit conditions ist, werden diese logisch wie hier angegeben verknüpft. Default: 'AND' + Returns: + dict: + - deleted, int: Anzahl der gelöschten Datensätze + """ + if self.check_collection_is_iban(collection): + # Directly update IBAN collection + return self._delete(collection, condition, multi) + + # Update all IBANs in group + update_result = 0 + for iban in self.get_group_ibans(collection): + update_result += self._delete(iban, condition, multi).get('deleted', 0) + + return {'deleted': update_result} + + def _delete(self, collection: str, condition: dict | list[dict], multi: str): + """ + Private Methode zum Löschen von Datensätzen in der Datenbank, + die die angegebene Bedingung erfüllen. Siehe 'delete' Methode. + Returns: dict: - deleted, int: Anzahl der gelöschten Datensätze diff --git a/handler/MongoDb.py b/handler/MongoDb.py index 5974d2c..ab7c8dd 100644 --- a/handler/MongoDb.py +++ b/handler/MongoDb.py @@ -112,7 +112,7 @@ def _insert(self, data: dict|list[dict], collection: str): except pymongo.errors.BulkWriteError: return {'inserted': 0} - def update(self, data, collection, condition=None, multi='AND', merge=True): + def _update(self, data, collection, condition=None, multi='AND', merge=True): """ Aktualisiert Datensätze in der Datenbank, die die angegebene Bedingung erfüllen. @@ -161,7 +161,7 @@ def update(self, data, collection, condition=None, multi='AND', merge=True): update_result = collection.update_many(query, update_op) return {'updated': update_result.modified_count} - def delete(self, collection, condition=None, multi='AND'): + def _delete(self, collection, condition=None, multi='AND'): """ Löscht Datensätze in der Datenbank, die die angegebene Bedingung erfüllen. diff --git a/handler/TinyDb.py b/handler/TinyDb.py index 17c332a..9314780 100644 --- a/handler/TinyDb.py +++ b/handler/TinyDb.py @@ -149,7 +149,7 @@ def _insert(self, data: dict|list[dict], collection: str): result = self.connection.table(collection).insert(data) return {'inserted': (1 if result else 0)} - def update(self, data, collection, condition=None, multi='AND', merge=True): + def _update(self, data, collection, condition=None, multi='AND', merge=True): """ Aktualisiert Datensätze in der Datenbank, die die angegebene Bedingung erfüllen. @@ -177,6 +177,8 @@ def update(self, data, collection, condition=None, multi='AND', merge=True): # No match, no update return { 'updated': 0 } + collection = self.connection.table(collection) + # care about the right format if data.get('tags') is not None and not isinstance(data.get('tags'), list): data['tags'] = [data.get('tags')] @@ -195,23 +197,14 @@ def update(self, data, collection, condition=None, multi='AND', merge=True): else: query = self._form_complete_query(condition, multi) - if not merge and self.check_collection_is_iban(collection): + if not merge: # Update all at once (no merging) - collection = self.connection.table(collection) update_result += collection.update(data, query) return { 'updated': len(update_result) } - if not merge and not self.check_collection_is_iban(collection): - # Update all at once (no merging) but loop ibans in group - for c in self.get_group_ibans(collection): - collection = self.connection.table(c) - update_result += collection.update(data, query) - return { 'updated': len(update_result) } - # Update every Entry one-by-one (merge every list item) for doc in docs_to_update: - collection = self.connection.table(doc.get('iban')) # Look for lists to merge with this entry for d in data.keys(): @@ -232,7 +225,7 @@ def update(self, data, collection, condition=None, multi='AND', merge=True): return { 'updated': len(update_result) } - def delete(self, collection, condition=None, multi='AND'): + def _delete(self, collection, condition=None, multi='AND'): """ Löscht Datensätze in der Datenbank, die die angegebene Bedingung erfüllen. From f20238b33b9aefc4898891b8784c113b796d70c6 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Thu, 5 Feb 2026 21:35:10 +0100 Subject: [PATCH 19/25] Fix Tests in MongoDB --- README.md | 6 ++++++ handler/MongoDb.py | 40 +++++++++++++++++++++++++------------- tests/test_integ_basics.py | 4 ++-- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 710339d..cdc0aec 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,12 @@ Daher sollte man beachten: In diesem Repository werden nur Basis-Regeln mitgeliefert, da speziellere und genauere Regeln sehr individuell auf einzelne Personen zugeschnitten sind. So schreibt zum Beispiel eine Versicherung die Versichertennummer mit in die Abbuchungen, was einen sehr guten Tagging-Indikator darstellt, jedoch nur für einen speziellen Nutzer dieses Programms. Das schreiben eigener Regeln ist daher unumgänglich, um bessere Ergebnisse zu erzielen. +### Wahl der Datenbankengine + +TinyDB sollte nur bei kleinen Instanzen mit einzelnen Benutzern gewählt werden. Ein paralleler Zugriff ist mit PynanceParser zwar möglich, allerdings sinkt die Performance und die Fehleranfälligkeit steigt mit der Anzahl der Requests und der Anzahl der Einträge in der Datenbank. Insbesondere bei I/O-schwacher Hardware (z.B. Raspberry mit SD Karte) kann es schnell zum Crash des Servers kommen. + +Für die produktive Nutzung wird MongoDB daher empfohlen! + ## Anpassungen / Contribution **You're Welcome !** :tada: diff --git a/handler/MongoDb.py b/handler/MongoDb.py index ab7c8dd..146a3cb 100644 --- a/handler/MongoDb.py +++ b/handler/MongoDb.py @@ -135,27 +135,41 @@ def _update(self, data, collection, condition=None, multi='AND', merge=True): - updated, int: Anzahl der aktualisierten Datensätze """ collection = self.connection[collection] + if collection is None: + logging.error(f"Collection {collection} not found for update!") + return {'updated': 0} # Form condition into a query query = self._form_complete_query(condition, multi) - # Handle Tag-Lists - new_tags = data.get('tags') - if new_tags and merge: - # care about the right format - if not isinstance(new_tags, list): - data['tags'] = [new_tags] + # care about the right format + if not isinstance(data.get('tags', []), list): + data['tags'] = [data['tags']] - # Clean $set data from tags - del data['tags'] + # Form condition into a query + if condition is None and merge: + logging.error('Using "merge" without a query is not possible') + return { 'error': 'Using "merge" without a query is not possible', 'updated': 0 } + + if merge: + # Care about lists and merge with special command + unmerged_data = {} + add_to_set = {} + for d in data.keys(): + + if isinstance(data[d], list): + add_to_set[d] = {'$each': data[d]} + continue - # Define Operation - update_op = { - '$set': data, - '$addToSet': {'tags': {'$each': new_tags}} - } + unmerged_data[d] = data[d] + + # Set all list keys to operation (or not if none) + update_op = {'$set': unmerged_data} + if add_to_set: + update_op['$addToSet'] = add_to_set else: + # No special handling update_op = {'$set': data} update_result = collection.update_many(query, update_op) diff --git a/tests/test_integ_basics.py b/tests/test_integ_basics.py index 4b24cdc..e2554ee 100644 --- a/tests/test_integ_basics.py +++ b/tests/test_integ_basics.py @@ -555,11 +555,11 @@ def test_tag_manual(test_app): # Check Tagging within a Group new_tag = { - 'tags': ['Tagged in Group'], + 'tags': ['Group-Tag'], 'overwrite': True } r = client.put( - "/api/setManualTag/testgroup/786e1d4e16832aa321a0176c854fe087", + "/api/setManualTag/testgroup/cf1fb4e6c131570e4f3b2ac857dead40", json=new_tag ) r = r.json From 126f2a1c3830d69e3be40bd9f62a2181cd39a6e3 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Fri, 6 Feb 2026 21:39:59 +0100 Subject: [PATCH 20/25] Add Empty Taglist Filter --- Models.md | 1 + app/static/js/functions.js | 4 ++-- app/templates/macros.html | 3 ++- app/ui.py | 2 +- handler/MongoDb.py | 2 ++ handler/TinyDb.py | 6 ++++++ settings/rule/00-default-tags.json | 2 +- tests/test_integ_basics.py | 34 +++++++++++++++++++++++++++--- 8 files changed, 46 insertions(+), 8 deletions(-) diff --git a/Models.md b/Models.md index e0947c1..021fd51 100644 --- a/Models.md +++ b/Models.md @@ -157,6 +157,7 @@ Dabei haben die Operatoren folgende Bedeutung: - `in`: Mindestens ein Wert der Vergleichsliste muss in dem Listenwert aus der Datenbank vorkommen. - `all`: Alle Werte der Vergleichsliste müssen in dem Listenwert aus der Datenbank vorkommen. - `notin`: Kein Wert der Vergleichsliste darf in dem Listenwert aus der Datenbank vorkommen. +- `exact` : Alle Werte der Vergleichsliste und keine anderen müssen in dem Listenwert aus der Datenbank vorkommen (unabhängig von der Reihenfolge). - `regex`: Regex String, der auf den Buchungstext angewendet werden soll. Ein Teil-Treffer des RegExes wird als Treffer gewertet. #### .tags, list (nur bei metatype: `rule`) diff --git a/app/static/js/functions.js b/app/static/js/functions.js index a2ae3d3..25ccdd3 100644 --- a/app/static/js/functions.js +++ b/app/static/js/functions.js @@ -64,10 +64,10 @@ function getFilteredList() { } const tags = document.getElementById('filter-tag-result').value; - if (tags) { + const tag_mode = document.getElementById('filter-tag-mode').value; + if (tags || tag_mode == 'exact') { query_args = query_args + arg_concat + 'tags=' + tags; arg_concat = '&'; - const tag_mode = document.getElementById('filter-tag-mode').value; if (tag_mode) { query_args = query_args + arg_concat + 'tag_mode=' + tag_mode; arg_concat = '&'; diff --git a/app/templates/macros.html b/app/templates/macros.html index a8c7db3..dad77a4 100644 --- a/app/templates/macros.html +++ b/app/templates/macros.html @@ -19,7 +19,7 @@ {{ transaction.date_tx | ctime }} ({{ transaction.valuta | ctime }}) - {{ transaction.text_tx[:90] }}{% if transaction.text_tx|length > 60 %} ...{%endif%} + {{ transaction.text_tx[:90] }}{% if transaction.text_tx|length > 90 %} ...{%endif%} {% if transaction.category %} @@ -123,6 +123,7 @@

Transaktionen filtern

+ diff --git a/app/ui.py b/app/ui.py index 3fea933..5520774 100644 --- a/app/ui.py +++ b/app/ui.py @@ -139,7 +139,7 @@ def filter_to_condition(self, get_args: dict) -> list: # Filter for Tags tag_filter = get_args.get('tags') if tag_filter is not None: - tag_filter = [t.strip() for t in tag_filter.split(',')] + tag_filter = [t.strip() for t in tag_filter.split(',') if t] condition.append({ 'key': 'tags', 'value': tag_filter, diff --git a/handler/MongoDb.py b/handler/MongoDb.py index 146a3cb..0f8e079 100644 --- a/handler/MongoDb.py +++ b/handler/MongoDb.py @@ -299,6 +299,8 @@ def _form_condition(self, condition): stmt = {'$nin': condition.get('value')} if condition_method == 'all': stmt = {'$all': condition.get('value')} + if condition_method == 'exact': + stmt = {'$all': condition.get('value'), '$size': len(condition.get('value'))} # Nested or Plain Key condition_key = condition.get('key') diff --git a/handler/TinyDb.py b/handler/TinyDb.py index 9314780..965cb52 100644 --- a/handler/TinyDb.py +++ b/handler/TinyDb.py @@ -354,6 +354,8 @@ def test_contains(value, search): where_statement = where_statement.test(self._none_of_test, condition_val) if condition_method == 'all': where_statement = where_statement.all(condition_val) + if condition_method == 'exact': + where_statement = where_statement.test(self._same_elements, condition_val) # Standard Query try: @@ -452,6 +454,10 @@ def _none_of_test(self, value, forbidden_values): """Benutzerdefinierter Test: Keines der Elemente ist in einer Liste vorhanden""" return not any(item in forbidden_values for item in value) + def _same_elements(self, value, given_values): + """Benutzerdefinierter Test: Alle Elemente der Listen sind gleich""" + return set(value) == set(given_values) + def _get_collections(self): """ Liste alle tables der Datenbank. diff --git a/settings/rule/00-default-tags.json b/settings/rule/00-default-tags.json index 2be4429..e0124e7 100644 --- a/settings/rule/00-default-tags.json +++ b/settings/rule/00-default-tags.json @@ -15,7 +15,7 @@ "tags": ["Lebensmittel"], "filter": [{ "key": "text_tx", - "value": "((? Date: Sat, 7 Feb 2026 23:37:29 +0100 Subject: [PATCH 21/25] UI polishing, more Rules and Parsers (Parser Regexes noch fixen) --- app/routes.py | 5 +++++ app/templates/macros.html | 2 +- handler/BaseDb.py | 2 ++ settings/parser/00-default.json | 7 ++++++- settings/rule/00-default-tags.json | 20 +++++++++++++++----- 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/app/routes.py b/app/routes.py index c21ada4..c5c942e 100644 --- a/app/routes.py +++ b/app/routes.py @@ -101,6 +101,7 @@ def welcome() -> str: ibans = parent.db_handler.list_ibans() groups = parent.db_handler.list_groups() meta = parent.db_handler.filter_metadata(condition=None) + meta.sort(key=lambda m: (m.get('metatype'), m.get('name'))) return render_template('index.html', ibans=ibans, groups=groups, meta=meta) @current_app.route('/', methods=['GET']) @@ -159,6 +160,8 @@ def iban(iban) -> str: if t not in tags: tags.append(t) + tags.sort() + # All distinct Categories # (must be filtered on our own because TinyDB doesn't support 'distinct' queries) cats = [] @@ -168,6 +171,8 @@ def iban(iban) -> str: if c and c not in cats: cats.append(c) + cats.sort() + return render_template('iban.html', transactions=rows[:entries_per_page], IBAN=iban, tags=tags, categories=cats, rules=rulenames, filters=frontend_filters) diff --git a/app/templates/macros.html b/app/templates/macros.html index dad77a4..f6e9fdb 100644 --- a/app/templates/macros.html +++ b/app/templates/macros.html @@ -123,7 +123,7 @@

Transaktionen filtern

- + diff --git a/handler/BaseDb.py b/handler/BaseDb.py index 4ab0b02..9f91f02 100644 --- a/handler/BaseDb.py +++ b/handler/BaseDb.py @@ -364,6 +364,7 @@ def list_ibans(self): """ all_collections = self._get_collections() ibans = [col for col in all_collections if self.check_collection_is_iban(col)] + ibans.sort() return ibans def list_groups(self): @@ -386,6 +387,7 @@ def list_groups(self): for group in meta_results: groups.append(group.get('groupname')) + groups.sort() return groups def _get_collections(self): diff --git a/settings/parser/00-default.json b/settings/parser/00-default.json index 92545c5..b93ddc9 100644 --- a/settings/parser/00-default.json +++ b/settings/parser/00-default.json @@ -2,7 +2,12 @@ { "name": "Mandatsreferenz", "metatype": "parser", - "regex": "Mandatsref\\:\\s?([A-z0-9]*)" + "regex": "((?<=MREF\\:\\s|Mandatsref\\:\\s)([A-z0-9]*))" + }, + { + "name": "End-toEnd-Referenz", + "metatype": "parser", + "regex": "((?<=EREF\\:\\s|End-to-End-Ref\\.\\:\\s)([A-Za-z0-9\\-\\s])+\\b(?!\\b:))" }, { "name": "Gläubiger-ID", diff --git a/settings/rule/00-default-tags.json b/settings/rule/00-default-tags.json index e0124e7..b6c481f 100644 --- a/settings/rule/00-default-tags.json +++ b/settings/rule/00-default-tags.json @@ -15,7 +15,7 @@ "tags": ["Lebensmittel"], "filter": [{ "key": "text_tx", - "value": "((? Date: Sun, 8 Feb 2026 22:23:54 +0100 Subject: [PATCH 22/25] =?UTF-8?q?Parser=20verbsessert=20und=20EREF=20hinzu?= =?UTF-8?q?gef=C3=BCgt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Models.md | 2 +- handler/Tags.py | 2 +- settings/parser/00-default.json | 6 ++-- tests/helper.py | 33 ++++++++++++++++++ tests/test_unit_parser.py | 59 +++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 tests/test_unit_parser.py diff --git a/Models.md b/Models.md index 021fd51..6400b5e 100644 --- a/Models.md +++ b/Models.md @@ -77,7 +77,7 @@ Frei wählbarer Name der Regel. #### .regex, r-str -Regex String, der auf den Buchungstext angewendet werden soll. Er muss genau eine Matching-Group enthalten. Der Wert dieses Treffers (der Gruppe) wird als Wert mit dem Namen der Regel in der Transaktion als Ergebnis gespeichert. +Regex String, der auf den Buchungstext angewendet werden soll. Es wird immer die erste Matching-Group übernommen. Der Wert dieses Treffers (der Gruppe) wird als Wert mit dem Namen der Regel in der Transaktion als Ergebnis gespeichert. ## Rule Objects diff --git a/handler/Tags.py b/handler/Tags.py index 0a2276d..11a78b8 100644 --- a/handler/Tags.py +++ b/handler/Tags.py @@ -35,7 +35,7 @@ def parse(self, input_data): for name, regex in parses.items(): re_match = regex.search(d['text_tx']) if re_match: - d['parsed'][name] = re_match.group(1) + d['parsed'][name] = re_match.group(1).strip() return input_data diff --git a/settings/parser/00-default.json b/settings/parser/00-default.json index b93ddc9..bcda06c 100644 --- a/settings/parser/00-default.json +++ b/settings/parser/00-default.json @@ -2,12 +2,12 @@ { "name": "Mandatsreferenz", "metatype": "parser", - "regex": "((?<=MREF\\:\\s|Mandatsref\\:\\s)([A-z0-9]*))" + "regex": "(?:MREF\\:\\s|Mandatsref\\:\\s)([A-z0-9]+)\\b(?!\\b:)" }, { - "name": "End-toEnd-Referenz", + "name": "End-to-End-Referenz", "metatype": "parser", - "regex": "((?<=EREF\\:\\s|End-to-End-Ref\\.\\:\\s)([A-Za-z0-9\\-\\s])+\\b(?!\\b:))" + "regex": "(?:EREF:\\s|End-to-End-Ref\\.\\:\\s)([A-Za-z0-9\\-\\s]+)\\b(?!\\b:)" }, { "name": "Gläubiger-ID", diff --git a/tests/helper.py b/tests/helper.py index 0b7e804..eb03337 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -1,7 +1,15 @@ """Hilfsfunktionen für die Tests""" import os +import sys import json +import re + +# Add Parent for importing from Modules +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(parent_dir) + +from handler.Tags import Tagger def check_transaktion_list(tx_list): @@ -340,3 +348,28 @@ def filter_metadata(self, condition, *args, **kwargs): # pylint: disable=unused- } ] return [] + +class MockTaggerParser(Tagger): + """ + Mock Tagger Parser zum Testen der Parser-Regeln + """ + + def __init__(self, rule_name): + """Konstruktor hinterlegt Variablen""" + self.rule_name = rule_name + super().__init__(MockDatabase()) + + def _load_parsers(self): + """Lädt eine spezielle Rgel aus der JSON""" + json_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + '..', 'settings', 'parser', '00-default.json' + ) + with open(json_path, "r") as f: + parser_settings = json.load(f) + + for p in parser_settings: + if p.get('name') == self.rule_name: + return {p.get('name'): re.compile(p.get('regex'))} + + assert False, f"Parser Regel {self.rule_name} konnte nicht geladen werden" \ No newline at end of file diff --git a/tests/test_unit_parser.py b/tests/test_unit_parser.py new file mode 100644 index 0000000..37726be --- /dev/null +++ b/tests/test_unit_parser.py @@ -0,0 +1,59 @@ +#!/usr/bin/python3 # pylint: disable=invalid-name +""" +Testmodul für das einzelne Testen von allen Parser-Regexes +""" + +import os +import sys + +# Add Parent for importing from Modules +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(parent_dir) + +from helper import MockTaggerParser + + +# Global test strings +test_string1 = ( + "Stadt Halle 0000005112 OBJEKT 0001 ABGABEN LT. " + "BESCHEID EREF: 2023-01-00111-9090-0000005112 " + "MREF: M1111111 Gläubiger-ID: DE7000100000077777 " + "SEPA-BASISLASTSCHRIFT wiederholend" +) +test_string2 = ( + "Stadt Halle 0000005112 OBJEKT 0001 ABGABEN LT. " + "BESCHEID End-to-End-Ref.: 2023-01-00111-9090-0000005112 " + "Mandatsref: M1111111 Gläubiger-ID: DE7000100000077777 " + "SEPA-BASISLASTSCHRIFT wiederholend" +) + +def test_parser_mandatsreferenz(test_app): + """ + Testet die Mandatsreferenz-Regex + """ + # Load specific parser rule + rule_name = "Mandatsreferenz" + tagger = MockTaggerParser(rule_name) + + # Matching + result = tagger.parse([{'parsed': {}, 'text_tx': test_string1}])[0] + assert result.get('parsed', {}).get(rule_name) == "M1111111" + + result = tagger.parse([{'parsed': {}, 'text_tx': test_string2}])[0] + assert result.get('parsed', {}).get(rule_name) == "M1111111" + + +def test_parser_mandatsreferenz(test_app): + """ + Testet die End-to-End-Regex + """ + # Load specific parser rule + rule_name = "End-to-End-Referenz" + tagger = MockTaggerParser(rule_name) + + # Matching + result = tagger.parse([{'parsed': {}, 'text_tx': test_string1}])[0] + assert result.get('parsed', {}).get(rule_name) == "2023-01-00111-9090-0000005112" + + result = tagger.parse([{'parsed': {}, 'text_tx': test_string2}])[0] + assert result.get('parsed', {}).get(rule_name) == "2023-01-00111-9090-0000005112" From 47c9167d903d7303a1d32a67da6bf991f9fa685d Mon Sep 17 00:00:00 2001 From: Pitastic Date: Tue, 10 Feb 2026 21:02:02 +0100 Subject: [PATCH 23/25] Rules: Strom, Wasser, Gas, Telefon --- settings/rule/00-default-tags.json | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/settings/rule/00-default-tags.json b/settings/rule/00-default-tags.json index b6c481f..0a17e2d 100644 --- a/settings/rule/00-default-tags.json +++ b/settings/rule/00-default-tags.json @@ -74,5 +74,45 @@ "value": "((^|\\W)ARAL\\s|(^|\\W)ESSO\\s|(^|\\W)SHELL\\s|(^|\\W)Shell\\s|AUTOHAUS|AUTOSERVICE|TANKSTELLE|(^|\\W)TANKEN\\W)", "compare": "regex" }] + }, + { + "metatype": "rule", + "name": "Gasverbrauch", + "tags": ["Gas"], + "filter": [{ + "key": "text_tx", + "value": "(\\sGAS\\sGmbH|Gasversorgung|Gasvertrag|Gasabrechnung|Gasrechnung)", + "compare": "regex" + }] + }, + { + "metatype": "rule", + "name": "Wasserverbrauch", + "tags": ["Wasser"], + "filter": [{ + "key": "text_tx", + "value": "(Wasserwerke|Wasserverband|Wasserbetriebe)", + "compare": "regex" + }] + }, + { + "metatype": "rule", + "name": "Stromverbrauch", + "tags": ["Strom"], + "filter": [{ + "key": "text_tx", + "value": "(Avacon\\sNetz\\sGmbH|EWE\\sNETZ\\sGmbH)", + "compare": "regex" + }] + }, + { + "metatype": "rule", + "name": "Telefon/Internet", + "tags": ["Kommunikation"], + "filter": [{ + "key": "text_tx", + "value": "((Telefonica\\sGermany\\s|htp\\s)GmbH|(Vodafone|Telekom\\sDeutschland\\sGmbH|1&1\\sIonos\\sSE|Deutsche\\sTelekom\\sAG))", + "compare": "regex" + }] } ] \ No newline at end of file From 91de43b3ec54daa931529942f16246e68c90c1ac Mon Sep 17 00:00:00 2001 From: Pitastic Date: Tue, 10 Feb 2026 21:11:34 +0100 Subject: [PATCH 24/25] letzte allgemeine Tags --- settings/rule/00-default-tags.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/settings/rule/00-default-tags.json b/settings/rule/00-default-tags.json index 0a17e2d..6e9bec4 100644 --- a/settings/rule/00-default-tags.json +++ b/settings/rule/00-default-tags.json @@ -114,5 +114,25 @@ "value": "((Telefonica\\sGermany\\s|htp\\s)GmbH|(Vodafone|Telekom\\sDeutschland\\sGmbH|1&1\\sIonos\\sSE|Deutsche\\sTelekom\\sAG))", "compare": "regex" }] + }, + { + "metatype": "rule", + "name": "Versicherungen allgemein", + "tags": ["Versicherung"], + "filter": [{ + "key": "text_tx", + "value": "([Vv]ersicherung|[Vv]ersicherungsvertrag)", + "compare": "regex" + }] + }, + { + "metatype": "rule", + "name": "Nahverkerhr/ÖPNV", + "tags": ["ÖPNV"], + "filter": [{ + "key": "text_tx", + "value": "(Nahverkehr|ÖPNV|Deutsche\\sBahn|[Vv]ekehrsbetriebe)", + "compare": "regex" + }] } ] \ No newline at end of file From d5cb6e54c61e2e7de0298cfe729b2572bc90e1f0 Mon Sep 17 00:00:00 2001 From: Pitastic Date: Wed, 11 Feb 2026 21:18:24 +0100 Subject: [PATCH 25/25] Letzte Rule --- settings/rule/00-default-tags.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/settings/rule/00-default-tags.json b/settings/rule/00-default-tags.json index 6e9bec4..0706000 100644 --- a/settings/rule/00-default-tags.json +++ b/settings/rule/00-default-tags.json @@ -134,5 +134,21 @@ "value": "(Nahverkehr|ÖPNV|Deutsche\\sBahn|[Vv]ekehrsbetriebe)", "compare": "regex" }] + }, + { + "metatype": "rule", + "name": "Bargeldabhebungen", + "tags": ["bar"], + "multi": "OR", + "filter": [{ + "key": "text_tx", + "value": "((BANK|[Bb]ank).*\\sGA\\s[0-9]+)|(^GA\\s(NR)?[0-9]+)", + "compare": "regex" + }, + { + "key": "art", + "value": "^Auszahlung", + "compare": "regex" + }] } ] \ No newline at end of file