Cleanup old tokens CLI

deploy/deploy.bash

@@ -38,6 +38,10 @@ DRONES_RULE_UNLOCK_TIMER_FILE="drones-rule-unlock.timer"
 DRONES_RULE_CLEAN_LARGE_JOURNALS_SERVICE_FILE="drones-clean-large-journals.service"
 DRONES_RULE_CLEAN_LARGE_JOURNALS_TIMER_FILE="drones-clean-large-journals.timer"
 
+# Drones token rules
+DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE="drones-clean-old-tokens.service"
+DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE="drones-clean-old-tokens.timer"
+
 # Drones for Great Wyrm
 DRONES_GREAT_WYRM_VOTES_SERVICE_FILE="drones-great-wyrm-votes.service"
 DRONES_GREAT_WYRM_VOTES_TIMER_FILE="drones-great-wyrm-votes.timer"
@@ -109,6 +113,16 @@ cp "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_LARGE_JOURNALS_TIMER_FILE}" "/home/ubuntu/
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${DRONES_RULE_CLEAN_LARGE_JOURNALS_TIMER_FILE}"
 
+# Tokens
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing Drones clean old tokens rule service and timer with: ${DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE}, ${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE}" "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${DRONES_RULE_CLEAN_OLD_TOKENS_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${DRONES_RULE_CLEAN_OLD_TOKENS_TIMER_FILE}"
+
 # Great Wyrm
 echo
 echo

deploy/drones-clean-old-tokens.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Clean old tokens from the database
+After=network.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/drones
+EnvironmentFile=/home/ubuntu/drones-secrets/app.env
+ExecStart=/home/ubuntu/drones-env/bin/python -m drones.cli cleanup tokens --max-lifetime 60
+SyslogIdentifier=drones-clean-old-tokens

deploy/drones-clean-old-tokens.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Clean old tokens from the database
+
+[Timer]
+OnBootSec=120s
+OnUnitActiveSec=23h
+
+[Install]
+WantedBy=timers.target

drones/cli.py

@@ -1,40 +1,38 @@
 import argparse
+import logging
+import time
+from contextlib import contextmanager
 from datetime import datetime, timedelta
 from distutils.util import strtobool
-from contextlib import contextmanager
-import logging
-from typing import List, Optional, Generator
+from typing import Generator, List, Optional
 from uuid import UUID
-import time
 
-from brood.external import SessionLocal as session_local_brood
+from brood.db import BROOD_DB_URI
+from brood.db import SessionLocal as session_local_brood
+from brood.db import create_brood_engine
+from brood.models import Token, TokenType, User
 from brood.settings import BUGOUT_URL
 from spire.db import (
-    SessionLocal as session_local_spire,
-    create_spire_engine,
-    SPIRE_DB_URI,
-    BUGOUT_SPIRE_THREAD_DB_POOL_SIZE,
     BUGOUT_SPIRE_THREAD_DB_MAX_OVERFLOW,
+    BUGOUT_SPIRE_THREAD_DB_POOL_SIZE,
     SPIRE_DB_POOL_RECYCLE_SECONDS,
+    SPIRE_DB_URI,
 )
-from spire.journal.models import JournalEntryLock, JournalEntry, Journal
+from spire.db import SessionLocal as session_local_spire
+from spire.db import create_spire_engine
+from spire.journal.models import Journal, JournalEntry, JournalEntryLock
 from sqlalchemy import func, text
 from sqlalchemy.orm import sessionmaker
 
-from .data import (
-    StatsTypes,
-    TimeScales,
-    RedisPickCommand,
-)
-from . import reports
-from . import statistics
+from . import reports, statistics
+from .data import RedisPickCommand, StatsTypes, TimeScales
+from .humbug_reports import pick_humbug_tasks_queue, process_humbug_tasks_queue
 from .migrations import ACTIONS, MIGRATIONS
-from .humbug_reports import process_humbug_tasks_queue, pick_humbug_tasks_queue
 from .settings import (
+    REDIS_FAILED_REPORTS_QUEUE,
+    REDIS_REPORTS_QUEUE,
     REPORTS_CHUNK_SIZE,
     WAITING_UNTIL_NEW_REPORTS,
-    REDIS_REPORTS_QUEUE,
-    REDIS_FAILED_REPORTS_QUEUE,
 )
 
 logging.basicConfig(level=logging.INFO)
@@ -252,6 +250,41 @@ def journal_rules_unlock_handler(args: argparse.Namespace) -> None:
         db_session_spire.close()
 
 
+def tokens_cleanup_handler(args: argparse.Namespace) -> None:
+    """
+    Clean old tokens with params:
+
+    token_type: bugout
+    created_at: <= max-lifetime
+    autogenerated user: false
+    restricted: false
+    """
+    custom_engine = create_brood_engine(
+        url=BROOD_DB_URI,
+        pool_size=BUGOUT_SPIRE_THREAD_DB_POOL_SIZE,
+        statement_timeout=300000,
+    )
+
+    today_diff = datetime.today() - timedelta(days=args.max_lifetime)
+    try:
+        with yield_session_maker(engine=custom_engine) as db_session:
+            # Filter autogenerated users should exist as additional query,
+            # otherwise join not supports sqlalchemy .delete(synchronize_session=False)
+            real_users = db_session.query(User.id).filter(User.autogenerated == False)
+            old_tokens = (
+                db_session.query(Token)
+                .filter(Token.user_id.in_(real_users))
+                .filter(Token.token_type == TokenType.bugout)
+                .filter(Token.restricted == False)
+                .filter(Token.created_at <= today_diff)
+                .delete(synchronize_session=False)
+            )
+            logger.info(f"Amount of deleted old tokens: {old_tokens}")
+            db_session.commit()
+    except Exception as err:
+        logger.error(f"Unable to clean old tokens, error: {str(err)}")
+
+
 def journal_entries_cleanup_handler(args: argparse.Namespace) -> None:
     """
     Clean entries from journal.
@@ -554,7 +587,6 @@ def main() -> None:
     subcommands_cleanup = parser_cleanup.add_subparsers(
         description="Drone cleanup commands"
     )
-
     parser_cleanup_journals = subcommands_cleanup.add_parser(
         "journals", description="Clean up journals"
     )
@@ -570,9 +602,19 @@ def main() -> None:
         default=10000,
         help="Number of entries to delete in one batch",
     )
-
     parser_cleanup_journals.set_defaults(func=journal_entries_cleanup_handler)
 
+    parser_cleanup_tokens = subcommands_cleanup.add_parser(
+        "tokens", description="Clean up tokens"
+    )
+    parser_cleanup_tokens.add_argument(
+        "--max-lifetime",
+        type=int,
+        required=True,
+        help="Max lifetime of token in days",
+    )
+    parser_cleanup_tokens.set_defaults(func=tokens_cleanup_handler)
+
     parser_cleanup_reports = subcommands_cleanup.add_parser(
         "reports", description="Clean up reports"
     )

drones/version.py

@@ -2,4 +2,4 @@
 Drones library and API version.
 """
 
-DRONES_VERSION = "0.1.5"
+DRONES_VERSION = "0.1.6"

sample.env

@@ -20,6 +20,7 @@ export REDIS_REPORTS_QUEUE="<redis key to humbug reports queue>"
 export REDIS_FAILED_REPORTS_QUEUE="<redis queue with reports which pushed to database with err>"
 export BUGOUT_HUMBUG_REDIS_TIMEOUT="0.5"
 export BUGOUT_HUMBUG_REDIS_CONNECTIONS_PER_PROCESS="10"
+export BUGOUT_APPLICATION_ID_HEADER="x-bugout-application-id"
 
 # Great Wyrm
 export BUGOUT_GREAT_WYRM_VOTES_JOURNAL_ID="<id_of_votes_humbug_journal>"

setup.py

@@ -30,11 +30,12 @@
     zip_safe=False,
     install_requires=[
         "boto3",
+        "bugout-brood>=0.3.1",
+        "bugout-spire>=0.4.2",
         "fastapi",
         "redis",
         "pydantic",
         "sendgrid",
-        "bugout-spire>=0.4.2",
         "sqlalchemy",
         "tqdm",
     ],