diff --git a/deploy/deploy.bash b/deploy/deploy.bash index eee9b4a..c936398 100755 --- a/deploy/deploy.bash +++ b/deploy/deploy.bash @@ -38,6 +38,10 @@ DRONES_RULE_UNLOCK_TIMER_FILE="drones-rule-unlock.timer" DRONES_RULE_CLEAN_LARGE_JOURNALS_SERVICE_FILE="drones-clean-large-journals.service" DRONES_RULE_CLEAN_LARGE_JOURNALS_TIMER_FILE="drones-clean-large-journals.timer" +# Drones token rules +DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE="drones-clean-old-tokens.service" +DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE="drones-clean-old-tokens.timer" + # Drones for Great Wyrm DRONES_GREAT_WYRM_VOTES_SERVICE_FILE="drones-great-wyrm-votes.service" DRONES_GREAT_WYRM_VOTES_TIMER_FILE="drones-great-wyrm-votes.timer" @@ -109,6 +113,16 @@ cp "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_LARGE_JOURNALS_TIMER_FILE}" "/home/ubuntu/ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${DRONES_RULE_CLEAN_LARGE_JOURNALS_TIMER_FILE}" +# Tokens +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Drones clean old tokens rule service and timer with: ${DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE}, ${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE}" "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}" +cp "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}" + # Great Wyrm echo echo diff --git a/deploy/drones-clean-old-tokens.service b/deploy/drones-clean-old-tokens.service new file mode 100644 index 0000000..6965374 --- /dev/null +++ b/deploy/drones-clean-old-tokens.service @@ -0,0 +1,10 @@ +[Unit] +Description=Clean old tokens from the database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/drones +EnvironmentFile=/home/ubuntu/drones-secrets/app.env +ExecStart=/home/ubuntu/drones-env/bin/python -m drones.cli cleanup tokens --max-lifetime 60 +SyslogIdentifier=drones-clean-old-tokens \ No newline at end of file diff --git a/deploy/drones-clean-old-tokens.timer b/deploy/drones-clean-old-tokens.timer new file mode 100644 index 0000000..1e2f263 --- /dev/null +++ b/deploy/drones-clean-old-tokens.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Clean old tokens from the database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=23h + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/drones/cli.py b/drones/cli.py index 0dff2f5..2452536 100644 --- a/drones/cli.py +++ b/drones/cli.py @@ -1,40 +1,38 @@ import argparse +import logging +import time +from contextlib import contextmanager from datetime import datetime, timedelta from distutils.util import strtobool -from contextlib import contextmanager -import logging -from typing import List, Optional, Generator +from typing import Generator, List, Optional from uuid import UUID -import time -from brood.external import SessionLocal as session_local_brood +from brood.db import BROOD_DB_URI +from brood.db import SessionLocal as session_local_brood +from brood.db import create_brood_engine +from brood.models import Token, TokenType, User from brood.settings import BUGOUT_URL from spire.db import ( - SessionLocal as session_local_spire, - create_spire_engine, - SPIRE_DB_URI, - BUGOUT_SPIRE_THREAD_DB_POOL_SIZE, BUGOUT_SPIRE_THREAD_DB_MAX_OVERFLOW, + BUGOUT_SPIRE_THREAD_DB_POOL_SIZE, SPIRE_DB_POOL_RECYCLE_SECONDS, + SPIRE_DB_URI, ) -from spire.journal.models import JournalEntryLock, JournalEntry, Journal +from spire.db import SessionLocal as session_local_spire +from spire.db import create_spire_engine +from spire.journal.models import Journal, JournalEntry, JournalEntryLock from sqlalchemy import func, text from sqlalchemy.orm import sessionmaker -from .data import ( - StatsTypes, - TimeScales, - RedisPickCommand, -) -from . import reports -from . import statistics +from . import reports, statistics +from .data import RedisPickCommand, StatsTypes, TimeScales +from .humbug_reports import pick_humbug_tasks_queue, process_humbug_tasks_queue from .migrations import ACTIONS, MIGRATIONS -from .humbug_reports import process_humbug_tasks_queue, pick_humbug_tasks_queue from .settings import ( + REDIS_FAILED_REPORTS_QUEUE, + REDIS_REPORTS_QUEUE, REPORTS_CHUNK_SIZE, WAITING_UNTIL_NEW_REPORTS, - REDIS_REPORTS_QUEUE, - REDIS_FAILED_REPORTS_QUEUE, ) logging.basicConfig(level=logging.INFO) @@ -252,6 +250,41 @@ def journal_rules_unlock_handler(args: argparse.Namespace) -> None: db_session_spire.close() +def tokens_cleanup_handler(args: argparse.Namespace) -> None: + """ + Clean old tokens with params: + + token_type: bugout + created_at: <= max-lifetime + autogenerated user: false + restricted: false + """ + custom_engine = create_brood_engine( + url=BROOD_DB_URI, + pool_size=BUGOUT_SPIRE_THREAD_DB_POOL_SIZE, + statement_timeout=300000, + ) + + today_diff = datetime.today() - timedelta(days=args.max_lifetime) + try: + with yield_session_maker(engine=custom_engine) as db_session: + # Filter autogenerated users should exist as additional query, + # otherwise join not supports sqlalchemy .delete(synchronize_session=False) + real_users = db_session.query(User.id).filter(User.autogenerated == False) + old_tokens = ( + db_session.query(Token) + .filter(Token.user_id.in_(real_users)) + .filter(Token.token_type == TokenType.bugout) + .filter(Token.restricted == False) + .filter(Token.created_at <= today_diff) + .delete(synchronize_session=False) + ) + logger.info(f"Amount of deleted old tokens: {old_tokens}") + db_session.commit() + except Exception as err: + logger.error(f"Unable to clean old tokens, error: {str(err)}") + + def journal_entries_cleanup_handler(args: argparse.Namespace) -> None: """ Clean entries from journal. @@ -554,7 +587,6 @@ def main() -> None: subcommands_cleanup = parser_cleanup.add_subparsers( description="Drone cleanup commands" ) - parser_cleanup_journals = subcommands_cleanup.add_parser( "journals", description="Clean up journals" ) @@ -570,9 +602,19 @@ def main() -> None: default=10000, help="Number of entries to delete in one batch", ) - parser_cleanup_journals.set_defaults(func=journal_entries_cleanup_handler) + parser_cleanup_tokens = subcommands_cleanup.add_parser( + "tokens", description="Clean up tokens" + ) + parser_cleanup_tokens.add_argument( + "--max-lifetime", + type=int, + required=True, + help="Max lifetime of token in days", + ) + parser_cleanup_tokens.set_defaults(func=tokens_cleanup_handler) + parser_cleanup_reports = subcommands_cleanup.add_parser( "reports", description="Clean up reports" ) diff --git a/drones/version.py b/drones/version.py index 716e416..60fbf30 100644 --- a/drones/version.py +++ b/drones/version.py @@ -2,4 +2,4 @@ Drones library and API version. """ -DRONES_VERSION = "0.1.5" +DRONES_VERSION = "0.1.6" diff --git a/sample.env b/sample.env index 44f57e1..b5a4c7e 100644 --- a/sample.env +++ b/sample.env @@ -20,6 +20,7 @@ export REDIS_REPORTS_QUEUE="" export REDIS_FAILED_REPORTS_QUEUE="" export BUGOUT_HUMBUG_REDIS_TIMEOUT="0.5" export BUGOUT_HUMBUG_REDIS_CONNECTIONS_PER_PROCESS="10" +export BUGOUT_APPLICATION_ID_HEADER="x-bugout-application-id" # Great Wyrm export BUGOUT_GREAT_WYRM_VOTES_JOURNAL_ID="" diff --git a/setup.py b/setup.py index 83b068f..d470c0a 100644 --- a/setup.py +++ b/setup.py @@ -30,11 +30,12 @@ zip_safe=False, install_requires=[ "boto3", + "bugout-brood>=0.3.1", + "bugout-spire>=0.4.2", "fastapi", "redis", "pydantic", "sendgrid", - "bugout-spire>=0.4.2", "sqlalchemy", "tqdm", ],