fix: Make Mongo threadsafe

helm-chart/sefaria/templates/configmap/gunicorn.yaml

@@ -32,12 +32,16 @@ data:
         max_requests = int(os.getenv("MAX_REQUESTS"))
         max_requests_jitter = int(os.getenv("MAX_REQUESTS_JITTER"))
 
-    {{- if .Values.instrumentation.enabled }}
     def post_fork(server, worker):
         server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+        # Reset MongoDB connection for fork safety
+        from sefaria.system.database import reset_connection
+        reset_connection()
+
+        {{- if .Values.instrumentation.enabled }}
         from opentelemetry.instrumentation.auto_instrumentation import sitecustomize
-
-    {{- end }}
+        {{- end }}
 
     def on_starting(server):
         from reader.startup import init_library_cache

sefaria/system/database.py

@@ -2,13 +2,17 @@
 database.py -- connection to MongoDB
 The system attribute _called_from_test is set in the py.test conftest.py file
 """
+import os
 import sys
 import pymongo
 import urllib.parse
 from pymongo.errors import OperationFailure
 
+import structlog
 from sefaria.settings import *
 
+logger = structlog.get_logger(__name__)
+
 def check_db_exists(db_name):
     dbnames = client.list_database_names()
     return db_name in dbnames
@@ -32,9 +36,9 @@ def get_test_db():
     #If we have jsut a single instance mongo (such as for development) the MONGO_HOST param should contain jsut the host string e.g "localhost")
     if MONGO_REPLICASET_NAME is None:
         if SEFARIA_DB_USER and SEFARIA_DB_PASSWORD:
-            client = pymongo.MongoClient(MONGO_HOST, MONGO_PORT, username=SEFARIA_DB_USER, password=SEFARIA_DB_PASSWORD)
+            client = pymongo.MongoClient(MONGO_HOST, MONGO_PORT, username=SEFARIA_DB_USER, password=SEFARIA_DB_PASSWORD, connect=False)
         else:
-            client = pymongo.MongoClient(MONGO_HOST, MONGO_PORT)
+            client = pymongo.MongoClient(MONGO_HOST, MONGO_PORT, connect=False)
     #Else if we are using a replica set mongo, we need to connect with a URI that containts a comma separated list of 'host:port' strings
     else:
         if SEFARIA_DB_USER and SEFARIA_DB_PASSWORD:
@@ -45,7 +49,7 @@ def get_test_db():
         else:
             connection_uri = 'mongodb://{}/?ssl=false&readPreference=primaryPreferred&replicaSet={}'.format(MONGO_HOST, MONGO_REPLICASET_NAME)
         # Now connect to the mongo server
-        client = pymongo.MongoClient(connection_uri)
+        client = pymongo.MongoClient(connection_uri, connect=False)
 
 
 
@@ -62,6 +66,52 @@ def drop_test():
     client.drop_database(TEST_DB)
 
 
+def reset_connection():
+    """
+    Reset MongoDB connection after fork. Called from Gunicorn post_fork hook.
+    Closes inherited connections and re-establishes fresh ones for this worker.
+    """
+    global client, db
+
+    pid = os.getpid()
+
+    # Close the inherited client to release file descriptors
+    if client is not None:
+        try:
+            client.close()
+            logger.info("mongodb_connection_closed", pid=pid)
+        except Exception as e:
+            logger.warning("mongodb_close_error", pid=pid, error=str(e))
+
+    # Create fresh client for this worker
+    if MONGO_REPLICASET_NAME is None:
+        if SEFARIA_DB_USER and SEFARIA_DB_PASSWORD:
+            client = pymongo.MongoClient(MONGO_HOST, MONGO_PORT,
+                                         username=SEFARIA_DB_USER,
+                                         password=SEFARIA_DB_PASSWORD,
+                                         connect=False)
+        else:
+            client = pymongo.MongoClient(MONGO_HOST, MONGO_PORT, connect=False)
+    else:
+        if SEFARIA_DB_USER and SEFARIA_DB_PASSWORD:
+            username = urllib.parse.quote_plus(SEFARIA_DB_USER)
+            password = urllib.parse.quote_plus(SEFARIA_DB_PASSWORD)
+            connection_uri = 'mongodb://{}:{}@{}/?ssl=false&readPreference=primaryPreferred&replicaSet={}'.format(
+                username, password, MONGO_HOST, MONGO_REPLICASET_NAME)
+        else:
+            connection_uri = 'mongodb://{}/?ssl=false&readPreference=primaryPreferred&replicaSet={}'.format(
+                MONGO_HOST, MONGO_REPLICASET_NAME)
+        client = pymongo.MongoClient(connection_uri, connect=False)
+
+    # Re-establish db reference
+    if not hasattr(sys, '_called_from_test'):
+        db = client[SEFARIA_DB]
+    else:
+        db = client[TEST_DB]
+
+    logger.info("mongodb_connection_reset", pid=pid)
+
+
 # Not used
 # def refresh_test():
 #     global client