Skip to content

⚡ Bolt: Optimize Lending Service N+1 queries with aggregation #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Woschj wants to merge 1 commit into
base: main
Choose a base branch
from
Open

⚡ Bolt: Optimize Lending Service N+1 queries with aggregation #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .jules/bolt.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,7 @@
## 2025-05-15 - [Optimization with Aggregation Pipelines]
**Learning:** This codebase frequently uses N+1 query patterns in service methods (e.g., looping through results and calling find_one). These can be significantly optimized using MongoDB aggregation pipelines with $lookup. However, mongomock (used in the test suite) has limited support for advanced $lookup features like 'let' and sub-pipelines.
**Action:** Use simple $lookup (localField/foreignField) when possible to maintain test compatibility, and handle any additional filtering or data processing in Python if necessary, which still provides a massive performance win by reducing database roundtrips to 1.

## 2026-02-26 - [N+1 Query Optimization in Lending Service]
**Learning:** Found N+1 query bottlenecks in `LendingService` methods (`get_active_lendings`, `get_recent_consumable_usage`, etc.) where related tool and worker data were fetched in loops. These methods were also being manually replicated with loops in the `manual_lending` route.
**Action:** Replace manual loops with MongoDB aggregation pipelines using `$lookup` and `$unwind`. Ensure the route uses these optimized service methods instead of implementing its own loops. Use `$unwind` without `preserveNullAndEmptyArrays` to replicate "inner join" filtering logic when required.
60 changes: 24 additions & 36 deletions app/routes/admin/system.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,47 +121,35 @@ def manual_lending():
# Verbrauchsmaterialien laden
consumables = mongodb.find('consumables', {'deleted': {'$ne': True}}, sort=[('name', 1)])

# Hole aktuelle Ausleihen
# Hole aktuelle Ausleihen (Bolt ⚡ Optimiert mit LendingService Aggregation zur Vermeidung von N+1 Problemen)
current_lendings = []
from app.services.lending_service import LendingService

# Aktuelle Werkzeug-Ausleihen
active_tool_lendings = mongodb.find('lendings', {'returned_at': None})
# Aktuelle Werkzeug-Ausleihen (Bolt ⚡ N+1 Fix)
active_tool_lendings = LendingService.get_active_lendings()
for lending in active_tool_lendings:
tool = mongodb.find_one('tools', {'barcode': lending['tool_barcode']})
worker = mongodb.find_one('workers', {'barcode': lending['worker_barcode']})

if tool and worker:
current_lendings.append({
'item_name': tool['name'],
'item_barcode': tool['barcode'],
'worker_name': f"{worker['firstname']} {worker['lastname']}",
'worker_barcode': worker['barcode'],
'action_date': lending['lent_at'],
'category': 'Werkzeug',
'amount': None
})

# Aktuelle Verbrauchsmaterial-Ausgaben (letzte 30 Tage)
thirty_days_ago = datetime.now() - timedelta(days=30)
recent_consumable_usages = mongodb.find('consumable_usages', {
'used_at': {'$gte': thirty_days_ago},
'quantity': {'$lt': 0} # Nur Ausgaben (negative Werte), nicht Entnahmen
})
current_lendings.append({
'item_name': lending.get('tool_name', 'Unbekanntes Werkzeug'),
'item_barcode': lending.get('tool_barcode', ''),
'worker_name': lending.get('worker_name', 'Unbekannter Mitarbeiter'),
'worker_barcode': lending.get('worker_barcode', ''),
'action_date': lending.get('lent_at'),
'category': 'Werkzeug',
'amount': None
})

# Aktuelle Verbrauchsmaterial-Ausgaben der letzten 30 Tage (Bolt ⚡ N+1 Fix)
recent_consumable_usages = LendingService.get_recent_consumable_usage(limit=100, days=30, only_outputs=True)
for usage in recent_consumable_usages:
consumable = mongodb.find_one('consumables', {'barcode': usage['consumable_barcode']})
worker = mongodb.find_one('workers', {'barcode': usage['worker_barcode']})

if consumable and worker:
current_lendings.append({
'item_name': consumable['name'],
'item_barcode': consumable['barcode'],
'worker_name': f"{worker['firstname']} {worker['lastname']}",
'worker_barcode': worker['barcode'],
'action_date': usage['used_at'],
'category': 'Verbrauchsmaterial',
'amount': usage['quantity']
})
current_lendings.append({
'item_name': usage.get('consumable_name', 'Unbekanntes Material'),
'item_barcode': usage.get('consumable_barcode', ''),
'worker_name': usage.get('worker_name', 'Unbekannter Mitarbeiter'),
'worker_barcode': usage.get('worker_barcode', ''),
'action_date': usage.get('used_at'),
'category': 'Verbrauchsmaterial',
'amount': usage.get('quantity')
})

# Sortiere nach Datum (neueste zuerst)
def safe_date_key(lending):
Expand Down
226 changes: 132 additions & 94 deletions app/services/lending_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,54 +341,109 @@ def _process_consumable_lending(item_barcode: str, worker_barcode: str, action:

@staticmethod
def get_active_lendings() -> list:
"""Holt alle aktiven Ausleihen"""
"""
Holt alle aktiven Ausleihen (Bolt ⚡ Optimiert mit Aggregation zur Vermeidung von N+1 Problemen)
"""
try:
active_lendings = mongodb.find('lendings', {'returned_at': None})
# Aggregation-Pipeline zur Vermeidung von N+1 Problemen (Bolt ⚡)
pipeline = [
{'$match': {'returned_at': None}},
{
'$lookup': {
'from': 'tools',
'localField': 'tool_barcode',
'foreignField': 'barcode',
'as': 'tool_info'
}
},
{'$unwind': '$tool_info'},
{
'$lookup': {
'from': 'workers',
'localField': 'worker_barcode',
'foreignField': 'barcode',
'as': 'worker_info'
}
},
{'$unwind': '$worker_info'},
{'$sort': {'lent_at': -1}}
]

results = mongodb.aggregate('lendings', pipeline)

# Erweitere mit Tool- und Worker-Informationen
# Formatiere Ergebnisse für Rückwärtskompatibilität
enriched_lendings = []
for lending in active_lendings:
tool = mongodb.find_one('tools', {'barcode': lending['tool_barcode']})
worker = mongodb.find_one('workers', {'barcode': lending['worker_barcode']})
for lending in results:
tool = lending['tool_info']
worker = lending['worker_info']

if tool and worker:
enriched_lendings.append({
**lending,
'tool_name': tool['name'],
'worker_name': f"{worker['firstname']} {worker['lastname']}",
'lent_at': lending['lent_at']
})
enriched_lendings.append({
**lending,
'tool_name': tool.get('name', 'Unbekanntes Tool'),
'worker_name': f"{worker.get('firstname', '')} {worker.get('lastname', '')}".strip() or 'Unbekannter Worker',
'lent_at': lending.get('lent_at')
})

# Sortiere nach Datum (neueste zuerst)
enriched_lendings.sort(key=lambda x: x.get('lent_at', datetime.min), reverse=True)
return enriched_lendings

except Exception as e:
logger.error(f"Fehler beim Laden aktiver Ausleihen: [Interner Fehler]")
return []

@staticmethod
def get_recent_consumable_usage(limit: int = 10) -> list:
"""Holt die letzten Verbrauchsmaterial-Entnahmen"""
def get_recent_consumable_usage(limit: int = 10, days: int = None, only_outputs: bool = False) -> list:
"""
Holt die letzten Verbrauchsmaterial-Entnahmen (Bolt ⚡ Optimiert mit Aggregation)
"""
try:
recent_usages = mongodb.find('consumable_usages')
# Sortiere und limitiere
recent_usages.sort(key=lambda x: x.get('used_at', datetime.min), reverse=True)
recent_usages = recent_usages[:limit]

# Erweitere mit Consumable- und Worker-Informationen
from datetime import timedelta
match_query = {}
if days:
cutoff = datetime.now() - timedelta(days=days)
match_query['used_at'] = {'$gte': cutoff}

if only_outputs:
match_query['quantity'] = {'$lt': 0}

pipeline = [
{'$match': match_query},
{'$sort': {'used_at': -1}},
{'$limit': limit},
{
'$lookup': {
'from': 'consumables',
'localField': 'consumable_barcode',
'foreignField': 'barcode',
'as': 'consumable_info'
}
},
{'$unwind': '$consumable_info'},
{
'$lookup': {
'from': 'workers',
'localField': 'worker_barcode',
'foreignField': 'barcode',
'as': 'worker_info'
}
},
{'$unwind': '$worker_info'}
]

results = mongodb.aggregate('consumable_usages', pipeline)

# Formatiere Ergebnisse für Rückwärtskompatibilität
enriched_usages = []
for usage in recent_usages:
consumable = mongodb.find_one('consumables', {'barcode': usage['consumable_barcode']})
worker = mongodb.find_one('workers', {'barcode': usage['worker_barcode']})
for usage in results:
consumable = usage['consumable_info']
worker = usage['worker_info']

if consumable and worker:
enriched_usages.append({
'consumable_name': consumable['name'],
'quantity': usage['quantity'],
'worker_name': f"{worker['firstname']} {worker['lastname']}",
'used_at': usage['used_at']
})
enriched_usages.append({
**usage,
'consumable_name': consumable.get('name', 'Unbekanntes Material'),
'worker_name': f"{worker.get('firstname', '')} {worker.get('lastname', '')}".strip() or 'Unbekannter Worker',
'quantity': usage.get('quantity', 0),
'used_at': usage.get('used_at')
})

return enriched_usages

Expand All @@ -399,42 +454,32 @@ def get_recent_consumable_usage(limit: int = 10) -> list:
@staticmethod
def get_worker_consumable_history(worker_barcode: str) -> List[Dict[str, Any]]:
"""
Holt die Verbrauchsmaterial-Historie für einen Mitarbeiter

Args:
worker_barcode: Barcode des Mitarbeiters

Returns:
List[Dict]: Liste der Verbrauchsmaterial-Ausgaben
Holt die Verbrauchsmaterial-Historie für einen Mitarbeiter (Bolt ⚡ Optimiert)
"""
try:
# Hole alle Verbrauchsmaterial-Ausgaben des Mitarbeiters
usages = mongodb.find('consumable_usages', {'worker_barcode': worker_barcode})
pipeline = [
{'$match': {'worker_barcode': worker_barcode}},
{
'$lookup': {
'from': 'consumables',
'localField': 'consumable_barcode',
'foreignField': 'barcode',
'as': 'consumable_info'
}
},
{'$unwind': '$consumable_info'},
{'$sort': {'used_at': -1}}
]

results = mongodb.aggregate('consumable_usages', pipeline)

# Erweitere mit Consumable-Informationen
enriched_usages = []
for usage in usages:
consumable = mongodb.find_one('consumables', {'barcode': usage['consumable_barcode']})
if consumable:
usage['consumable_name'] = consumable.get('name', '')
usage['consumable_barcode'] = usage['consumable_barcode']
enriched_usages.append(usage)

# Sortiere nach Datum (neueste zuerst)
def safe_date_key(usage):
used_at = usage.get('used_at')
if isinstance(used_at, str):
try:
return datetime.strptime(used_at, '%Y-%m-%d %H:%M:%S')
except (ValueError, TypeError):
return datetime.min
elif isinstance(used_at, datetime):
return used_at
else:
return datetime.min
# Formatiere Ergebnisse
for usage in results:
consumable = usage['consumable_info']
usage['consumable_name'] = consumable.get('name', '')
usage['consumable_barcode'] = usage.get('consumable_barcode')

enriched_usages.sort(key=safe_date_key, reverse=True)
return enriched_usages
return results

except Exception as e:
logger.error(f"Fehler beim Laden der Verbrauchsmaterial-Historie: [Interner Fehler]")
Expand Down Expand Up @@ -477,41 +522,34 @@ def get_current_lending(tool_barcode: str) -> Optional[Dict[str, Any]]:
@staticmethod
def get_tool_lending_history(tool_barcode: str) -> List[Dict[str, Any]]:
"""
Holt die Ausleihhistorie für ein Werkzeug

Args:
tool_barcode: Barcode des Werkzeugs

Returns:
List[Dict]: Liste der Ausleihen
Holt die Ausleihhistorie für ein Werkzeug (Bolt ⚡ Optimiert)
"""
try:
lendings = mongodb.find('lendings', {'tool_barcode': tool_barcode})
pipeline = [
{'$match': {'tool_barcode': tool_barcode}},
{
'$lookup': {
'from': 'workers',
'localField': 'worker_barcode',
'foreignField': 'barcode',
'as': 'worker_info'
}
},
{'$unwind': {'path': '$worker_info', 'preserveNullAndEmptyArrays': True}},
{'$sort': {'lent_at': -1}}
]

results = mongodb.aggregate('lendings', pipeline)

# Erweitere mit Worker-Informationen
enriched_lendings = []
for lending in lendings:
worker = mongodb.find_one('workers', {'barcode': lending['worker_barcode']})
# Formatiere Ergebnisse
for lending in results:
worker = lending.get('worker_info')
if worker:
lending['worker_name'] = f"{worker['firstname']} {worker['lastname']}"

enriched_lendings.append(lending)

# Sortiere nach Datum (neueste zuerst)
def safe_date_key(lending):
lent_at = lending.get('lent_at')
if isinstance(lent_at, str):
try:
return datetime.strptime(lent_at, '%Y-%m-%d %H:%M:%S')
except (ValueError, TypeError):
return datetime.min
elif isinstance(lent_at, datetime):
return lent_at
lending['worker_name'] = f"{worker.get('firstname', '')} {worker.get('lastname', '')}".strip() or 'Unbekannter Worker'
else:
return datetime.min
lending['worker_name'] = 'Unbekannter Worker'

enriched_lendings.sort(key=safe_date_key, reverse=True)
return enriched_lendings
return results

except Exception as e:
logger.error(f"Fehler beim Laden der Ausleihhistorie: [Interner Fehler]")
Expand Down