From fc61f7a902bdc97fb8dd0c0b59eb7e06bc4d24ec Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 7 Dec 2025 19:54:30 -0600 Subject: [PATCH 01/21] Add handling of args in config file and improve keyFile support --- packages/mongodb-runner/src/cli.ts | 4 ++ packages/mongodb-runner/src/mongoserver.ts | 84 ++++++++++++++++++---- 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/packages/mongodb-runner/src/cli.ts b/packages/mongodb-runner/src/cli.ts index 6e00db37..c500284c 100644 --- a/packages/mongodb-runner/src/cli.ts +++ b/packages/mongodb-runner/src/cli.ts @@ -90,6 +90,10 @@ import type { MongoClientOptions } from 'mongodb'; .demandCommand(1, 'A command needs to be provided') .help().argv; const [command, ...args] = argv._.map(String); + // Allow args to be provided by the config file. + if (Array.isArray(argv.args)) { + args.push(...argv.args.map(String)); + } if (argv.debug || argv.verbose) { createDebug.enable('mongodb-runner'); } diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index 7e7fab1c..a231757b 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -20,6 +20,7 @@ import { debugVerbose, jsonClone, makeConnectionString, + sleep, } from './util'; /** @@ -286,9 +287,11 @@ export class MongoServer extends EventEmitter { logEntryStream.resume(); srv.port = port; - const buildInfoError = await srv._populateBuildInfo('insert-new'); - if (buildInfoError) { - debug('failed to get buildInfo', buildInfoError); + if (!options.args?.includes('--keyFile')) { + const buildInfoError = await srv._populateBuildInfo('insert-new'); + if (buildInfoError) { + debug('failed to get buildInfo', buildInfoError); + } } } catch (err) { await srv.close(); @@ -301,24 +304,77 @@ export class MongoServer extends EventEmitter { async updateDefaultConnectionOptions( options: Partial, ): Promise { + // Assume we need these new options to connect. + this.defaultConnectionOptions = { + ...this.defaultConnectionOptions, + ...options, + }; + + // If there is no auth in the connection options, do an immediate metadata refresh and return. let buildInfoError: Error | null = null; + if (!options.auth) { + buildInfoError = await this._populateBuildInfo('restore-check'); + if (buildInfoError) { + debug( + 'failed to refresh buildInfo when updating connection options', + buildInfoError, + options, + ); + throw buildInfoError; + } + return; + } + + debug('Waiting for authorization on', this.port); + + // Wait until we can get connectionStatus. + let supportsAuth = false; + let error: unknown = null; for (let attempts = 0; attempts < 10; attempts++) { - buildInfoError = await this._populateBuildInfo('restore-check', { - ...options, - }); - if (!buildInfoError) break; + error = null; + try { + supportsAuth = await this.withClient(async (client) => { + const status = await client + .db('admin') + .command({ connectionStatus: 1 }); + if (status.authInfo.authenticatedUsers.length > 0) { + return true; + } + // If the server does not support auth, just get the build info without + // setting the metadata. + debug('Server does not support authorization', this.port); + this.buildInfo = await client.db('admin').command({ buildInfo: 1 }); + return false; + }); + } catch (e) { + error = e; + await sleep(2 ** attempts * 10); + } + if (error === null) { + break; + } + } + + if (error !== null) { + throw error; + } + + if (!supportsAuth) { + return; + } + + const mode = this.hasInsertedMetadataCollEntry + ? 'restore-check' + : 'insert-new'; + buildInfoError = await this._populateBuildInfo(mode); + if (buildInfoError) { debug( - 'failed to get buildInfo when setting new options', + 'failed to refresh buildInfo when updating connection options', buildInfoError, options, - this.connectionString, ); + throw buildInfoError; } - if (buildInfoError) throw buildInfoError; - this.defaultConnectionOptions = { - ...this.defaultConnectionOptions, - ...options, - }; } async close(): Promise { From 3a6eadf95c20ad7c682ddd44a9482dfb3913de04 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 8 Dec 2025 12:12:20 -0600 Subject: [PATCH 02/21] handle requireApiVersion --- packages/mongodb-runner/src/mongocluster.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 20b0a4eb..7ced7188 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -110,6 +110,11 @@ export interface CommonOptions { */ tlsAddClientKey?: boolean; + /** + * Whether to require an API version for commands. + */ + requireApiVersion?: number; + /** * Topology of the cluster. */ @@ -528,6 +533,17 @@ export class MongoCluster extends EventEmitter { } await cluster.addAuthIfNeeded(); + + // Set up requireApiVersion if requested. + if (options.requireApiVersion !== undefined) { + await cluster.withClient(async (client) => { + const admin = client.db('admin'); + await admin.command({ setParameter: 1, requireApiVersion: true }); + }); + await cluster.updateDefaultConnectionOptions({ + serverApi: String(options.requireApiVersion) as '1', + }); + } return cluster; } From a3904c66abd6854bd643066bd851684611145640 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 8 Dec 2025 12:41:37 -0600 Subject: [PATCH 03/21] fix requireApiVersion handling --- packages/mongodb-runner/src/mongocluster.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 7ced7188..961311dd 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -536,10 +536,20 @@ export class MongoCluster extends EventEmitter { // Set up requireApiVersion if requested. if (options.requireApiVersion !== undefined) { - await cluster.withClient(async (client) => { - const admin = client.db('admin'); - await admin.command({ setParameter: 1, requireApiVersion: true }); - }); + if (options.topology === 'replset') { + throw new Error( + 'requireApiVersion is not supported for replica sets, see SERVER-97010', + ); + } + await Promise.all( + [...cluster.servers].map( + async (child) => + await child.withClient(async (client) => { + const admin = client.db('admin'); + await admin.command({ setParameter: 1, requireApiVersion: true }); + }), + ), + ); await cluster.updateDefaultConnectionOptions({ serverApi: String(options.requireApiVersion) as '1', }); From f52db8f5cdba36a9d573aac1f3d38ab93155e45e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 8 Dec 2025 13:08:03 -0600 Subject: [PATCH 04/21] do not set requireApiVersion in shards --- packages/mongodb-runner/src/mongocluster.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 961311dd..20c07a96 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -493,6 +493,7 @@ export class MongoCluster extends EventEmitter { ...options, ...s, topology: 'replset', + requireApiVersion: undefined, users: isConfig ? undefined : options.users, // users go on the mongos/config server only for the config set }); return [cluster, isConfig] as const; From 4d776e02bb9ef1aa1ba5c2099575f847d4de3d50 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 8 Dec 2025 14:57:50 -0600 Subject: [PATCH 05/21] allow connection string to be captured by the caller --- packages/mongodb-runner/src/cli.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/mongodb-runner/src/cli.ts b/packages/mongodb-runner/src/cli.ts index c500284c..633eb19a 100644 --- a/packages/mongodb-runner/src/cli.ts +++ b/packages/mongodb-runner/src/cli.ts @@ -115,22 +115,29 @@ import type { MongoClientOptions } from 'mongodb'; async function start() { const { cluster, id } = await utilities.start(argv, args); const cs = new ConnectionString(cluster.connectionString); - console.log(`Server started and running at ${cs.toString()}`); + // Only the connection string should print to stdout so it can be captured + // by a calling process. + console.error(`Server started and running at ${cs.toString()}`); if (cluster.oidcIssuer) { cs.typedSearchParams().set( 'authMechanism', 'MONGODB-OIDC', ); - console.log(`OIDC provider started and running at ${cluster.oidcIssuer}`); - console.log(`Server connection string with OIDC auth: ${cs.toString()}`); + console.error( + `OIDC provider started and running at ${cluster.oidcIssuer}`, + ); + console.error( + `Server connection string with OIDC auth: ${cs.toString()}`, + ); } - console.log('Run the following command to stop the instance:'); - console.log( + console.error('Run the following command to stop the instance:'); + console.error( `${argv.$0} stop --id=${id}` + (argv.runnerDir !== defaultRunnerDir ? `--runnerDir=${argv.runnerDir}` : ''), ); + console.log(cs.toString()); cluster.unref(); } From 892a8ba55dc4548577dfbe0f1cc5ceb28b9aafac Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 9 Dec 2025 08:34:41 -0600 Subject: [PATCH 06/21] address review --- packages/mongodb-runner/src/mongocluster.ts | 48 ++++++++++++--------- packages/mongodb-runner/src/mongoserver.ts | 6 ++- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 20c07a96..55c4fd80 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -534,27 +534,7 @@ export class MongoCluster extends EventEmitter { } await cluster.addAuthIfNeeded(); - - // Set up requireApiVersion if requested. - if (options.requireApiVersion !== undefined) { - if (options.topology === 'replset') { - throw new Error( - 'requireApiVersion is not supported for replica sets, see SERVER-97010', - ); - } - await Promise.all( - [...cluster.servers].map( - async (child) => - await child.withClient(async (client) => { - const admin = client.db('admin'); - await admin.command({ setParameter: 1, requireApiVersion: true }); - }), - ), - ); - await cluster.updateDefaultConnectionOptions({ - serverApi: String(options.requireApiVersion) as '1', - }); - } + await cluster.addRequireApiVersionIfNeeded(options); return cluster; } @@ -563,6 +543,32 @@ export class MongoCluster extends EventEmitter { yield* this.shards; } + async addRequireApiVersionIfNeeded({ + ...options + }: MongoClusterOptions): Promise { + // Set up requireApiVersion if requested. + if (options.requireApiVersion !== undefined) { + return; + } + if (options.topology === 'replset') { + throw new Error( + 'requireApiVersion is not supported for replica sets, see SERVER-97010', + ); + } + await Promise.all( + [...this.servers].map( + async (child) => + await child.withClient(async (client) => { + const admin = client.db('admin'); + await admin.command({ setParameter: 1, requireApiVersion: true }); + }), + ), + ); + await this.updateDefaultConnectionOptions({ + serverApi: String(options.requireApiVersion) as '1', + }); + } + async addAuthIfNeeded(): Promise { if (!this.users?.length) return; // Sleep to give time for a possible replset election to settle. diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index a231757b..1abb6a0b 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -287,6 +287,8 @@ export class MongoServer extends EventEmitter { logEntryStream.resume(); srv.port = port; + // If a keyFile is present, we cannot read or write on the server until + // a user is added to the primary. if (!options.args?.includes('--keyFile')) { const buildInfoError = await srv._populateBuildInfo('insert-new'); if (buildInfoError) { @@ -340,8 +342,8 @@ export class MongoServer extends EventEmitter { if (status.authInfo.authenticatedUsers.length > 0) { return true; } - // If the server does not support auth, just get the build info without - // setting the metadata. + // The server is most likely an arbiter, which does not support + // authenticated users but does support getting the buildInfo. debug('Server does not support authorization', this.port); this.buildInfo = await client.db('admin').command({ buildInfo: 1 }); return false; From 56a3abdfefccf41f85b7da2d7794a733039107ed Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 9 Dec 2025 09:28:35 -0600 Subject: [PATCH 07/21] fix logic --- packages/mongodb-runner/src/mongocluster.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index 55c4fd80..b5b93959 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -547,7 +547,7 @@ export class MongoCluster extends EventEmitter { ...options }: MongoClusterOptions): Promise { // Set up requireApiVersion if requested. - if (options.requireApiVersion !== undefined) { + if (options.requireApiVersion === undefined) { return; } if (options.topology === 'replset') { From 89894fcfa9f4931ffb92c1230c502637fa5ec079 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 19 Dec 2025 19:17:55 -0600 Subject: [PATCH 08/21] add tests --- packages/mongodb-runner/src/cli.spec.ts | 129 ++++++++++++++++++ .../mongodb-runner/src/mongocluster.spec.ts | 47 +++++++ packages/mongodb-runner/src/mongoserver.ts | 1 + .../mongodb-runner/test/fixtures/config.json | 38 ++++++ 4 files changed, 215 insertions(+) create mode 100644 packages/mongodb-runner/src/cli.spec.ts create mode 100644 packages/mongodb-runner/test/fixtures/config.json diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts new file mode 100644 index 00000000..25edde0c --- /dev/null +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -0,0 +1,129 @@ +import { expect } from 'chai'; +import { promises as fs } from 'fs'; +import path from 'path'; +import os from 'os'; +import { promisify } from 'util'; +import { execFile } from 'child_process'; +import createDebug from 'debug'; +import sinon from 'sinon'; +import type { LogEntry } from './mongologreader'; +import { MongoClient } from 'mongodb'; +import { eventually } from './util'; + +if (process.env.CI) { + createDebug.enable('mongodb-runner,mongodb-downloader'); +} + +const execFileAsync = promisify(execFile); +const tmpDir = path.join(os.tmpdir(), `runner-cli-tests-${Date.now()}`); + +async function runCli(args: string[]): Promise { + const { stdout } = await execFileAsync('mongodb-runner', args); + return stdout; +} + +describe('cli', function () { + this.timeout(1_000_000); // Downloading Windows binaries can take a very long time... + + before(async function () { + await fs.mkdir(tmpDir, { recursive: true }); + }); + + after(async function () { + await fs.rm(tmpDir, { + recursive: true, + maxRetries: 100, + }); + }); + + afterEach(function () { + sinon.restore(); + }); + + it('can manage a standalone cluster with command line args', async function () { + // Start the CLI with arguments and capture stdout. + const stdout = await runCli(['start', '--topology', 'standalone']); + + // stdout is JUST the connection string. + const connectionString = stdout.trim(); + expect(connectionString).to.match(/^mongodb(\+srv)?:\/\//); + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); + it('can manage a replset cluster with command line args', async function () { + const stdout = await runCli([ + 'start', + '--topology', + 'replset', + '--secondaries', + '2', + '--arbiters', + '1', + '--version', + '8.0.x', + '--', + '--replSet', + 'repl0', + ]); + const connectionString = stdout.trim(); + expect(/repl0/.test(connectionString)).to.be.true; + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); + it('can manage a sharded cluster with command line args', async function () { + const stdout = await runCli([ + 'start', + '--topology', + 'sharded', + '--shards', + '2', + '--version', + '7.0.x', + ]); + const connectionString = stdout.trim(); + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); + it.only('can manage a cluster with a config file', async function () { + const configFile = path.resolve( + __dirname, + '..', + 'test', + 'fixtures', + 'config.json', + ); + const stdout = await runCli(['start', '--config', configFile]); + const connectionString = stdout.trim(); + expect(/repl0/.test(connectionString)).to.be.true; + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); +}); diff --git a/packages/mongodb-runner/src/mongocluster.spec.ts b/packages/mongodb-runner/src/mongocluster.spec.ts index edcb20b0..122749a8 100644 --- a/packages/mongodb-runner/src/mongocluster.spec.ts +++ b/packages/mongodb-runner/src/mongocluster.spec.ts @@ -630,4 +630,51 @@ describe('MongoCluster', function () { { user: 'testuser', db: 'admin' }, ]); }); + it.only('can use a keyFile', async function () { + const keyFile = path.join(tmpDir, 'keyFile'); + await fs.writeFile(keyFile, 'secret', { mode: 0o400 }); + cluster = await MongoCluster.start({ + version: '8.x', + topology: 'replset', + tmpDir, + secondaries: 1, + arbiters: 1, + args: ['--keyFile', keyFile], + users: [ + { + username: 'testuser', + password: 'testpass', + roles: [ + { role: 'userAdminAnyDatabase', db: 'admin' }, + { role: 'clusterAdmin', db: 'admin' }, + ], + }, + ], + }); + expect(cluster.connectionString).to.be.a('string'); + expect(cluster.serverVersion).to.match(/^8\./); + expect(cluster.connectionString).to.include('testuser:testpass@'); + cluster = await MongoCluster.deserialize(cluster.serialize()); + expect(cluster.connectionString).to.include('testuser:testpass@'); + }); + it('can support requireApiVersion', async function () { + cluster = await MongoCluster.start({ + version: '8.x', + topology: 'sharded', + tmpDir, + secondaries: 1, + shards: 1, + requireApiVersion: 1, + args: ['--setParameter', 'enableTestCommands=1'], + }); + expect(cluster.connectionString).to.be.a('string'); + expect(cluster.serverVersion).to.match(/^8\./); + await cluster.withClient((client) => { + expect(client.serverApi?.version).to.eq('1'); + }); + cluster = await MongoCluster.deserialize(cluster.serialize()); + await cluster.withClient((client) => { + expect(client.serverApi?.version).to.eq('1'); + }); + }); }); diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index 1abb6a0b..eab2f9e0 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -340,6 +340,7 @@ export class MongoServer extends EventEmitter { .db('admin') .command({ connectionStatus: 1 }); if (status.authInfo.authenticatedUsers.length > 0) { + debug('Server supports authorization', this.port); return true; } // The server is most likely an arbiter, which does not support diff --git a/packages/mongodb-runner/test/fixtures/config.json b/packages/mongodb-runner/test/fixtures/config.json new file mode 100644 index 00000000..5235578a --- /dev/null +++ b/packages/mongodb-runner/test/fixtures/config.json @@ -0,0 +1,38 @@ +{ + "topology": "replset", + "args": ["--replSet", "repl0"], + "rsMembers": [ + { + "args": [ + "--oplogSize", + "500", + "--setParameter", + "enableTestCommands=true" + ], + "tags": { + "ordinal": "one", + "dc": "ny" + }, + "priority": 1 + }, + { + "args": [ + "--oplogSize", + "500", + "--setParameter", + "enableTestCommands=true" + ], + "tags": { + "ordinal": "two", + "dc": "pa" + }, + "priority": 1 + }, + { + "args": ["--setParameter", "enableTestCommands=true"], + "tags": {}, + "priority": 0, + "arbiterOnly": true + } + ] +} From 40bacb370c2f9e9ecb35e5f15ae73171afea80ba Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 19 Dec 2025 19:25:23 -0600 Subject: [PATCH 09/21] lint --- packages/mongodb-runner/src/cli.spec.ts | 4 +--- packages/mongodb-runner/src/mongocluster.spec.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index 25edde0c..49a0eb63 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -6,9 +6,7 @@ import { promisify } from 'util'; import { execFile } from 'child_process'; import createDebug from 'debug'; import sinon from 'sinon'; -import type { LogEntry } from './mongologreader'; import { MongoClient } from 'mongodb'; -import { eventually } from './util'; if (process.env.CI) { createDebug.enable('mongodb-runner,mongodb-downloader'); @@ -105,7 +103,7 @@ describe('cli', function () { // Call `stop` on the CLI await runCli(['stop', '--all']); }); - it.only('can manage a cluster with a config file', async function () { + it('can manage a cluster with a config file', async function () { const configFile = path.resolve( __dirname, '..', diff --git a/packages/mongodb-runner/src/mongocluster.spec.ts b/packages/mongodb-runner/src/mongocluster.spec.ts index 122749a8..8ff02ed4 100644 --- a/packages/mongodb-runner/src/mongocluster.spec.ts +++ b/packages/mongodb-runner/src/mongocluster.spec.ts @@ -630,7 +630,7 @@ describe('MongoCluster', function () { { user: 'testuser', db: 'admin' }, ]); }); - it.only('can use a keyFile', async function () { + it('can use a keyFile', async function () { const keyFile = path.join(tmpDir, 'keyFile'); await fs.writeFile(keyFile, 'secret', { mode: 0o400 }); cluster = await MongoCluster.start({ From 1c8843afe74eb51b3953b066a052204659fe0b5f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 19 Dec 2025 20:06:41 -0600 Subject: [PATCH 10/21] windows fix --- packages/mongodb-runner/src/cli.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index 49a0eb63..2777c0d4 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -16,7 +16,9 @@ const execFileAsync = promisify(execFile); const tmpDir = path.join(os.tmpdir(), `runner-cli-tests-${Date.now()}`); async function runCli(args: string[]): Promise { - const { stdout } = await execFileAsync('mongodb-runner', args); + const isWin = process.platform === 'win32'; + const runner = isWin ? 'mongodb-runner.cmd' : 'mongodb-runner'; + const { stdout } = await execFileAsync(runner, args); return stdout; } From d07df19a723fa2fcebb8dd6e16719fdd47436879 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 21 Dec 2025 10:42:07 -0600 Subject: [PATCH 11/21] windows fix and add oidc test --- packages/mongodb-runner/src/cli.spec.ts | 52 +++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index 2777c0d4..a1d28b16 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -4,6 +4,7 @@ import path from 'path'; import os from 'os'; import { promisify } from 'util'; import { execFile } from 'child_process'; +import type { ExecOptions } from 'child_process'; import createDebug from 'debug'; import sinon from 'sinon'; import { MongoClient } from 'mongodb'; @@ -15,10 +16,12 @@ if (process.env.CI) { const execFileAsync = promisify(execFile); const tmpDir = path.join(os.tmpdir(), `runner-cli-tests-${Date.now()}`); -async function runCli(args: string[]): Promise { - const isWin = process.platform === 'win32'; - const runner = isWin ? 'mongodb-runner.cmd' : 'mongodb-runner'; - const { stdout } = await execFileAsync(runner, args); +async function runCli( + args: string[], + options: ExecOptions = {}, +): Promise { + const fullArgs = ['mongodb-runner', ...args]; + const { stdout } = await execFileAsync('npx', fullArgs, options); return stdout; } @@ -54,6 +57,9 @@ describe('cli', function () { await client.close(); expect(result.ok).to.eq(1); + const lsStdout = await runCli(['ls']); + expect(lsStdout.includes(connectionString)).to.be.true; + // Call `stop` on the CLI await runCli(['stop', '--all']); }); @@ -123,6 +129,44 @@ describe('cli', function () { await client.close(); expect(result.ok).to.eq(1); + // Call `stop` on the CLI + await runCli(['stop', '--all']); + }); + it('can use mock oidc provider on linux', async function () { + if (process.platform !== 'linux') return this.skip(); + + // Start the CLI with arguments and capture stdout. + const stdout = await runCli( + [ + 'start', + '--topology', + 'standalone', + '--version', + '8.0.x-enterprise', + '--oidc', + '--port=0', + ], + { + env: { + ...process.env, + RUN_OIDC_MOCK_PROVIDER: '1', + }, + }, + ); + + // stdout is JUST the connection string. + const connectionString = stdout.trim(); + expect(connectionString).to.match(/^mongodb(\+srv)?:\/\//); + + // Connect to the cluster. + const client = new MongoClient(connectionString); + const result = await client.db('admin').command({ ping: 1 }); + await client.close(); + expect(result.ok).to.eq(1); + + const lsStdout = await runCli(['ls']); + expect(lsStdout.includes(connectionString)).to.be.true; + // Call `stop` on the CLI await runCli(['stop', '--all']); }); From cfca4f400b652eacd2a8d983c90ed63c4e4ec00a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 22 Dec 2025 07:36:26 -0600 Subject: [PATCH 12/21] clean up tests --- packages/mongodb-runner/src/cli.spec.ts | 69 ++++++++----------------- 1 file changed, 22 insertions(+), 47 deletions(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index a1d28b16..b32138dd 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -26,9 +26,13 @@ async function runCli( } describe('cli', function () { - this.timeout(1_000_000); // Downloading Windows binaries can take a very long time... + this.timeout(30_000); before(async function () { + if (process.platform === 'win32') { + // XXX: Skipping the CLI tests on Windows due to differences in spawn arguments. + return this.skip(); + } await fs.mkdir(tmpDir, { recursive: true }); }); @@ -49,7 +53,7 @@ describe('cli', function () { // stdout is JUST the connection string. const connectionString = stdout.trim(); - expect(connectionString).to.match(/^mongodb(\+srv)?:\/\//); + expect(connectionString).to.match(/^mongodb:\/\//); // Connect to the cluster. const client = new MongoClient(connectionString); @@ -57,11 +61,26 @@ describe('cli', function () { await client.close(); expect(result.ok).to.eq(1); + // Exercise the rest of the cli. const lsStdout = await runCli(['ls']); expect(lsStdout.includes(connectionString)).to.be.true; - // Call `stop` on the CLI await runCli(['stop', '--all']); + + await runCli(['prune']); + }); + it('can execute against a cluster', async function () { + const stdout = await runCli([ + 'exec', + '-t', + 'standalone', + '--', + 'sh', + '-c', + 'echo $MONGODB_URI', + ]); + const connectionString = stdout.trim(); + expect(connectionString).to.match(/^mongodb:\/\//); }); it('can manage a replset cluster with command line args', async function () { const stdout = await runCli([ @@ -81,13 +100,11 @@ describe('cli', function () { const connectionString = stdout.trim(); expect(/repl0/.test(connectionString)).to.be.true; - // Connect to the cluster. const client = new MongoClient(connectionString); const result = await client.db('admin').command({ ping: 1 }); await client.close(); expect(result.ok).to.eq(1); - // Call `stop` on the CLI await runCli(['stop', '--all']); }); it('can manage a sharded cluster with command line args', async function () { @@ -102,13 +119,11 @@ describe('cli', function () { ]); const connectionString = stdout.trim(); - // Connect to the cluster. const client = new MongoClient(connectionString); const result = await client.db('admin').command({ ping: 1 }); await client.close(); expect(result.ok).to.eq(1); - // Call `stop` on the CLI await runCli(['stop', '--all']); }); it('can manage a cluster with a config file', async function () { @@ -123,51 +138,11 @@ describe('cli', function () { const connectionString = stdout.trim(); expect(/repl0/.test(connectionString)).to.be.true; - // Connect to the cluster. const client = new MongoClient(connectionString); const result = await client.db('admin').command({ ping: 1 }); await client.close(); expect(result.ok).to.eq(1); - // Call `stop` on the CLI - await runCli(['stop', '--all']); - }); - it('can use mock oidc provider on linux', async function () { - if (process.platform !== 'linux') return this.skip(); - - // Start the CLI with arguments and capture stdout. - const stdout = await runCli( - [ - 'start', - '--topology', - 'standalone', - '--version', - '8.0.x-enterprise', - '--oidc', - '--port=0', - ], - { - env: { - ...process.env, - RUN_OIDC_MOCK_PROVIDER: '1', - }, - }, - ); - - // stdout is JUST the connection string. - const connectionString = stdout.trim(); - expect(connectionString).to.match(/^mongodb(\+srv)?:\/\//); - - // Connect to the cluster. - const client = new MongoClient(connectionString); - const result = await client.db('admin').command({ ping: 1 }); - await client.close(); - expect(result.ok).to.eq(1); - - const lsStdout = await runCli(['ls']); - expect(lsStdout.includes(connectionString)).to.be.true; - - // Call `stop` on the CLI await runCli(['stop', '--all']); }); }); From 1c7d928c21e837ff234f60beb63d214cae51d726 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 22 Dec 2025 09:28:06 -0600 Subject: [PATCH 13/21] windows fix --- packages/mongodb-runner/src/cli.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index b32138dd..185b2d9c 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -14,7 +14,6 @@ if (process.env.CI) { } const execFileAsync = promisify(execFile); -const tmpDir = path.join(os.tmpdir(), `runner-cli-tests-${Date.now()}`); async function runCli( args: string[], @@ -27,12 +26,14 @@ async function runCli( describe('cli', function () { this.timeout(30_000); + let tmpDir = ''; before(async function () { if (process.platform === 'win32') { // XXX: Skipping the CLI tests on Windows due to differences in spawn arguments. return this.skip(); } + tmpDir = path.join(os.tmpdir(), `runner-cli-tests-${Date.now()}`); await fs.mkdir(tmpDir, { recursive: true }); }); From 07dad9f93b6a270d0e81fefcbcedf9d3d7aa659a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 22 Dec 2025 10:20:05 -0600 Subject: [PATCH 14/21] windows fix --- packages/mongodb-runner/src/cli.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index 185b2d9c..42adf325 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -38,6 +38,10 @@ describe('cli', function () { }); after(async function () { + if (process.platform === 'win32') { + // XXX: Skipping the CLI tests on Windows due to differences in spawn arguments. + return this.skip(); + } await fs.rm(tmpDir, { recursive: true, maxRetries: 100, From c795428cfc191402d3ac29264dad19e89b589d5e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 22 Dec 2025 10:37:50 -0600 Subject: [PATCH 15/21] increase timeout --- packages/mongodb-runner/src/cli.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index 42adf325..fe0e8291 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -25,7 +25,7 @@ async function runCli( } describe('cli', function () { - this.timeout(30_000); + this.timeout(100_000); let tmpDir = ''; before(async function () { From c39f200a309cc8953f954bfedafda4836d979f44 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 22 Dec 2025 11:44:05 -0600 Subject: [PATCH 16/21] windows fix --- packages/mongodb-runner/src/cli.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index fe0e8291..99efbea2 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -40,7 +40,7 @@ describe('cli', function () { after(async function () { if (process.platform === 'win32') { // XXX: Skipping the CLI tests on Windows due to differences in spawn arguments. - return this.skip(); + return; } await fs.rm(tmpDir, { recursive: true, From 633b84704581e0d09e994e536a1f569e6e0949d1 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 27 Jan 2026 19:14:35 +0100 Subject: [PATCH 17/21] fixup: re-enable Windows in cli.spec.ts, formatting --- packages/mongodb-runner/src/cli.spec.ts | 30 +++++++------------ .../mongodb-runner/src/mongocluster.spec.ts | 2 ++ 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index 99efbea2..bbba1fa0 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -5,22 +5,19 @@ import os from 'os'; import { promisify } from 'util'; import { execFile } from 'child_process'; import type { ExecOptions } from 'child_process'; -import createDebug from 'debug'; -import sinon from 'sinon'; import { MongoClient } from 'mongodb'; -if (process.env.CI) { - createDebug.enable('mongodb-runner,mongodb-downloader'); -} - const execFileAsync = promisify(execFile); async function runCli( args: string[], options: ExecOptions = {}, ): Promise { - const fullArgs = ['mongodb-runner', ...args]; - const { stdout } = await execFileAsync('npx', fullArgs, options); + const { stdout } = await execFileAsync( + process.execPath, + [path.resolve(__dirname, '..', 'bin', 'runner.js'), ...args], + options, + ); return stdout; } @@ -29,29 +26,18 @@ describe('cli', function () { let tmpDir = ''; before(async function () { - if (process.platform === 'win32') { - // XXX: Skipping the CLI tests on Windows due to differences in spawn arguments. - return this.skip(); - } tmpDir = path.join(os.tmpdir(), `runner-cli-tests-${Date.now()}`); await fs.mkdir(tmpDir, { recursive: true }); }); after(async function () { - if (process.platform === 'win32') { - // XXX: Skipping the CLI tests on Windows due to differences in spawn arguments. - return; - } await fs.rm(tmpDir, { + force: true, recursive: true, maxRetries: 100, }); }); - afterEach(function () { - sinon.restore(); - }); - it('can manage a standalone cluster with command line args', async function () { // Start the CLI with arguments and capture stdout. const stdout = await runCli(['start', '--topology', 'standalone']); @@ -74,6 +60,7 @@ describe('cli', function () { await runCli(['prune']); }); + it('can execute against a cluster', async function () { const stdout = await runCli([ 'exec', @@ -87,6 +74,7 @@ describe('cli', function () { const connectionString = stdout.trim(); expect(connectionString).to.match(/^mongodb:\/\//); }); + it('can manage a replset cluster with command line args', async function () { const stdout = await runCli([ 'start', @@ -112,6 +100,7 @@ describe('cli', function () { await runCli(['stop', '--all']); }); + it('can manage a sharded cluster with command line args', async function () { const stdout = await runCli([ 'start', @@ -131,6 +120,7 @@ describe('cli', function () { await runCli(['stop', '--all']); }); + it('can manage a cluster with a config file', async function () { const configFile = path.resolve( __dirname, diff --git a/packages/mongodb-runner/src/mongocluster.spec.ts b/packages/mongodb-runner/src/mongocluster.spec.ts index 8ff02ed4..7c4b9a82 100644 --- a/packages/mongodb-runner/src/mongocluster.spec.ts +++ b/packages/mongodb-runner/src/mongocluster.spec.ts @@ -630,6 +630,7 @@ describe('MongoCluster', function () { { user: 'testuser', db: 'admin' }, ]); }); + it('can use a keyFile', async function () { const keyFile = path.join(tmpDir, 'keyFile'); await fs.writeFile(keyFile, 'secret', { mode: 0o400 }); @@ -657,6 +658,7 @@ describe('MongoCluster', function () { cluster = await MongoCluster.deserialize(cluster.serialize()); expect(cluster.connectionString).to.include('testuser:testpass@'); }); + it('can support requireApiVersion', async function () { cluster = await MongoCluster.start({ version: '8.x', From b5ddab1ed6a74aedbf64f20fb07ca7f7d853cb9e Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 27 Jan 2026 23:00:19 +0100 Subject: [PATCH 18/21] fixup: refactor arbiter/auth/keyfile handling --- packages/mongodb-runner/src/mongocluster.ts | 55 ++++-- packages/mongodb-runner/src/mongoserver.ts | 177 ++++++++++++-------- packages/mongodb-runner/src/tls-helpers.ts | 4 +- packages/mongodb-runner/src/util.ts | 24 +++ packages/mongodb-runner/tsconfig.json | 4 +- 5 files changed, 180 insertions(+), 84 deletions(-) diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index b5b93959..df3658fd 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -18,6 +18,7 @@ import { jsonClone, debugVerbose, makeConnectionString, + safePromiseAll, } from './util'; import { OIDCMockProviderProcess } from './oidc'; import { EventEmitter } from 'events'; @@ -327,10 +328,10 @@ export class MongoCluster extends EventEmitter { cluster.replSetName = serialized.replSetName; cluster.defaultConnectionOptions = serialized.defaultConnectionOptions; cluster.users = serialized.users; - cluster.servers = await Promise.all( + cluster.servers = await safePromiseAll( serialized.servers.map((srv: any) => MongoServer.deserialize(srv)), ); - cluster.shards = await Promise.all( + cluster.shards = await safePromiseAll( serialized.shards.map((shard: any) => MongoCluster.deserialize(shard)), ); cluster.oidcMockProviderProcess = serialized.oidcMockProviderProcess @@ -429,12 +430,13 @@ export class MongoCluster extends EventEmitter { ); assert.notStrictEqual(primaryIndex, -1); - const nodes = await Promise.all( + const nodes = await safePromiseAll( rsMembers.map(async (member) => { return [ await MongoServer.start({ ...options, args: member.args, + isArbiter: member.arbiterOnly ?? false, binary: 'mongod', }), member, @@ -486,7 +488,7 @@ export class MongoCluster extends EventEmitter { } else if (options.topology === 'sharded') { const { shards, mongosArgs } = processShardOptions(options); debug('starting config server and shard servers', shards); - const allShards = await Promise.all( + const allShards = await safePromiseAll( shards.map(async (s) => { const isConfig = s.args?.includes('--configsvr'); const cluster = await MongoCluster.start({ @@ -505,7 +507,7 @@ export class MongoCluster extends EventEmitter { .map(([shard]) => shard); cluster.shards.push(configsvr, ...shardsvrs); - const mongosServers: MongoServer[] = await Promise.all( + const mongosServers: MongoServer[] = await safePromiseAll( mongosArgs.map(async (args) => { debug('starting mongos'); return await MongoServer.start({ @@ -535,6 +537,14 @@ export class MongoCluster extends EventEmitter { await cluster.addAuthIfNeeded(); await cluster.addRequireApiVersionIfNeeded(options); + try { + await cluster.assertAllServersHaveInsertedLocalMetadata(); + } catch (err) { + // Allow connection errors if automatic tls client key addition is disabled + // since that option implies to users that they are taking responsibility for + // ensuring that the correct runner instance is used for managing processes. + if (options.tlsAddClientKey !== false) throw err; + } return cluster; } @@ -543,6 +553,19 @@ export class MongoCluster extends EventEmitter { yield* this.shards; } + private *allServers(): Iterable { + for (const child of this.servers) yield child; + for (const shard of this.shards) yield* shard.allServers(); + } + + async assertAllServersHaveInsertedLocalMetadata(): Promise { + await safePromiseAll( + [...this.allServers()].map( + async (server) => await server.assertHasInsertedLocalMetadata(), + ), + ); + } + async addRequireApiVersionIfNeeded({ ...options }: MongoClusterOptions): Promise { @@ -555,7 +578,7 @@ export class MongoCluster extends EventEmitter { 'requireApiVersion is not supported for replica sets, see SERVER-97010', ); } - await Promise.all( + await safePromiseAll( [...this.servers].map( async (child) => await child.withClient(async (client) => { @@ -578,7 +601,14 @@ export class MongoCluster extends EventEmitter { for (const user of this.users) { const { username, password, ...rest } = user; debug('adding new user', { username, ...rest }, this.connectionString); - await admin.command({ createUser: username, pwd: password, ...rest }); + await admin.command({ + createUser: username, + pwd: password, + // User management commands can only use w:1 or w:majority + // https://github.com/mongodb/mongo/blob/4b65e1c663042d6c2e879ab20ba4b3c22439997a/src/mongo/db/global_catalog/sharding_catalog_client_impl.cpp#L1112-L1113 + writeConcern: { w: 'majority', j: true, wtimeout: 0 }, + ...rest, + }); } }); await this.updateDefaultConnectionOptions({ @@ -589,7 +619,7 @@ export class MongoCluster extends EventEmitter { async updateDefaultConnectionOptions( options: Partial, ): Promise { - await Promise.all( + await safePromiseAll( [...this.children()].map(async (child) => child.updateDefaultConnectionOptions(options), ), @@ -601,7 +631,7 @@ export class MongoCluster extends EventEmitter { } async close(): Promise { - await Promise.all( + await safePromiseAll( [...this.children(), this.oidcMockProviderProcess].map((closable) => closable?.close(), ), @@ -640,7 +670,12 @@ export class MongoCluster extends EventEmitter { case 'standalone': return {}; case 'replset': - return { writeConcern: { w: this.servers.length, j: true } }; + return { + writeConcern: { + w: this.servers.filter((s) => !s.isArbiter).length, + j: true, + }, + }; case 'sharded': return { writeConcern: { diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index eab2f9e0..190c4087 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -41,6 +41,10 @@ export interface MongoServerOptions { docker?: string | string[]; /** Internal options for the MongoDB client used by this server instance. */ internalClientOptions?: Partial; + /** Internal option -- if this is an arbiter, it does not understand user auth */ + isArbiter?: boolean; + /** Internal option -- if keyfile auth is used, this will be its contents */ + keyFileContents?: string; } interface SerializedServerProperties { @@ -51,12 +55,24 @@ interface SerializedServerProperties { defaultConnectionOptions?: Partial; startTime: string; hasInsertedMetadataCollEntry: boolean; + isArbiter?: boolean; + keyFileContents?: string; } export interface MongoServerEvents { mongoLog: [LogEntry]; } +function getKeyFileOption(args?: string[] | undefined): string | undefined { + if (!args) return undefined; + const splitArgIndex = args.indexOf('--keyFile'); + if (splitArgIndex !== -1) { + return args[splitArgIndex + 1]; + } + const arg = args.find((a) => a.startsWith('--keyFile=')); + return arg?.split('=')[1]; +} + export class MongoServer extends EventEmitter { public uuid: string = uuid(); private buildInfo?: Document; @@ -67,6 +83,8 @@ export class MongoServer extends EventEmitter { private closing = false; private startTime = new Date().toISOString(); private hasInsertedMetadataCollEntry = false; + public isArbiter = false; + private keyFileContents?: string; private defaultConnectionOptions?: Partial; get id(): string { @@ -87,6 +105,8 @@ export class MongoServer extends EventEmitter { startTime: this.startTime, hasInsertedMetadataCollEntry: this.hasInsertedMetadataCollEntry, defaultConnectionOptions: jsonClone(this.defaultConnectionOptions ?? {}), + isArbiter: this.isArbiter, + keyFileContents: this.keyFileContents, }; } @@ -98,6 +118,8 @@ export class MongoServer extends EventEmitter { srv.port = serialized.port; srv.defaultConnectionOptions = serialized.defaultConnectionOptions; srv.closing = !!(await srv._populateBuildInfo('restore-check')); + srv.isArbiter = !!serialized.isArbiter; + srv.keyFileContents = serialized.keyFileContents; if (!srv.closing) { srv.pid = serialized.pid; srv.dbPath = serialized.dbPath; @@ -167,6 +189,16 @@ export class MongoServer extends EventEmitter { }: MongoServerOptions): Promise { const srv = new MongoServer(); srv.defaultConnectionOptions = { ...options.internalClientOptions }; + srv.isArbiter = !!options.isArbiter; + const keyFilePath = getKeyFileOption(options.args); + if (keyFilePath) { + srv.keyFileContents = await fs.readFile(keyFilePath, 'utf8'); + debug('read keyFile contents for server', { + keyFilePath, + uuid: srv.uuid, + }); + } + if (!options.docker) { const dbPath = path.join(options.tmpDir, `db-${srv.uuid}`); await fs.mkdir(dbPath, { recursive: true }); @@ -288,8 +320,10 @@ export class MongoServer extends EventEmitter { srv.port = port; // If a keyFile is present, we cannot read or write on the server until - // a user is added to the primary. - if (!options.args?.includes('--keyFile')) { + // a user is added to the primary. This information will be populated + // as part of the cluster's startup process when authenticating, + // enforced by its `assertAllServersHaveInsertedLocalMetadata()` check. + if (!keyFilePath) { const buildInfoError = await srv._populateBuildInfo('insert-new'); if (buildInfoError) { debug('failed to get buildInfo', buildInfoError); @@ -306,78 +340,25 @@ export class MongoServer extends EventEmitter { async updateDefaultConnectionOptions( options: Partial, ): Promise { - // Assume we need these new options to connect. - this.defaultConnectionOptions = { - ...this.defaultConnectionOptions, - ...options, - }; - - // If there is no auth in the connection options, do an immediate metadata refresh and return. let buildInfoError: Error | null = null; - if (!options.auth) { - buildInfoError = await this._populateBuildInfo('restore-check'); - if (buildInfoError) { - debug( - 'failed to refresh buildInfo when updating connection options', - buildInfoError, - options, - ); - throw buildInfoError; - } - return; - } - - debug('Waiting for authorization on', this.port); - - // Wait until we can get connectionStatus. - let supportsAuth = false; - let error: unknown = null; for (let attempts = 0; attempts < 10; attempts++) { - error = null; - try { - supportsAuth = await this.withClient(async (client) => { - const status = await client - .db('admin') - .command({ connectionStatus: 1 }); - if (status.authInfo.authenticatedUsers.length > 0) { - debug('Server supports authorization', this.port); - return true; - } - // The server is most likely an arbiter, which does not support - // authenticated users but does support getting the buildInfo. - debug('Server does not support authorization', this.port); - this.buildInfo = await client.db('admin').command({ buildInfo: 1 }); - return false; - }); - } catch (e) { - error = e; - await sleep(2 ** attempts * 10); - } - if (error === null) { - break; - } - } - - if (error !== null) { - throw error; - } - - if (!supportsAuth) { - return; - } - - const mode = this.hasInsertedMetadataCollEntry - ? 'restore-check' - : 'insert-new'; - buildInfoError = await this._populateBuildInfo(mode); - if (buildInfoError) { + buildInfoError = await this._populateBuildInfo('restore-check', { + ...options, + }); + if (!buildInfoError) break; debug( - 'failed to refresh buildInfo when updating connection options', + 'failed to get buildInfo when setting new options', buildInfoError, options, + this.connectionString, ); - throw buildInfoError; + await sleep(2 ** attempts * 10); } + if (buildInfoError) throw buildInfoError; + this.defaultConnectionOptions = { + ...this.defaultConnectionOptions, + ...options, + }; } async close(): Promise { @@ -413,6 +394,17 @@ export class MongoServer extends EventEmitter { mode: 'insert-new' | 'restore-check', ): Promise { const hello = await client.db('admin').command({ hello: 1 }); + const { arbiterOnly } = hello; + if (arbiterOnly === this.isArbiter) { + debug('skipping metadata check for arbiter'); + return; + } + if (this.isArbiter) { + throw new Error( + 'Arbiter flag mismatch -- server should be arbiter but hello indicates it is not', + ); + } + const isMongoS = hello.msg === 'isdbgrid'; const insertedInfo = pick(this.serialize(), [ '_id', @@ -471,6 +463,17 @@ export class MongoServer extends EventEmitter { try { // directConnection + retryWrites let us write to `local` db on secondaries clientOpts = { retryWrites: false, ...clientOpts }; + if ( + this.keyFileContents && + (clientOpts.auth || this.defaultConnectionOptions?.auth) + ) { + clientOpts.auth = { + username: '__system', + password: this.keyFileContents, + }; + clientOpts.authMechanism = 'SCRAM-SHA-256'; + clientOpts.authSource = 'local'; + } this.buildInfo = await this.withClient(async (client) => { // Insert the metadata entry, except if we're a freshly started mongos // (which does not have its own storage to persist) @@ -478,7 +481,11 @@ export class MongoServer extends EventEmitter { return await client.db('admin').command({ buildInfo: 1 }); }, clientOpts); } catch (err) { - debug('failed to get buildInfo, treating as closed server', err); + debug( + 'failed to get buildInfo, treating as closed server', + err, + new Error().stack, + ); return err as Error; } debug( @@ -501,11 +508,23 @@ export class MongoServer extends EventEmitter { fn: Fn, clientOptions: MongoClientOptions = {}, ): Promise> { - const client = await MongoClient.connect(this.connectionString, { + clientOptions = { directConnection: true, ...this.defaultConnectionOptions, ...clientOptions, - }); + }; + if ( + this.isArbiter && + clientOptions.auth && + clientOptions.auth.username !== '__system' + ) { + // Arbiters do not understand user authentication + clientOptions.auth = undefined; + } + const client = await MongoClient.connect( + this.connectionString, + clientOptions, + ); try { return await fn(client); } finally { @@ -524,4 +543,22 @@ export class MongoServer extends EventEmitter { (this.childProcess?.stdout as any)?.unref(); (this.childProcess?.stderr as any)?.unref(); } + + async assertHasInsertedLocalMetadata(): Promise { + if (!this.hasInsertedMetadataCollEntry) { + debug('populating metadata collection entry after initial setup'); + const err = await this._populateBuildInfo('insert-new'); + if (err) throw err; + } + if (!this.buildInfo) { + throw new Error( + `Server buildInfo is not populated ${JSON.stringify(this.serialize())}`, + ); + } + if (!this.hasInsertedMetadataCollEntry && !this.isArbiter) { + throw new Error( + `Server has not inserted metadata collection entry ${JSON.stringify(this.serialize())}`, + ); + } + } } diff --git a/packages/mongodb-runner/src/tls-helpers.ts b/packages/mongodb-runner/src/tls-helpers.ts index dd19985c..fed5ae5d 100644 --- a/packages/mongodb-runner/src/tls-helpers.ts +++ b/packages/mongodb-runner/src/tls-helpers.ts @@ -1,6 +1,6 @@ import * as x509 from '@peculiar/x509'; import { webcrypto } from 'crypto'; -import { uuid } from './util'; +import { safePromiseAll, uuid } from './util'; import path from 'path'; import { writeFile, readFile } from 'fs/promises'; import type { MongoClientOptions } from 'mongodb'; @@ -62,7 +62,7 @@ export async function handleTLSClientKeyOptions({ const clientPEM = path.join(tmpDir, `mongodb-runner-client-${id}.pem`); const caPEM = path.join(tmpDir, `mongodb-runner-ca-${id}.pem`); - await Promise.all([ + await safePromiseAll([ (async () => { await writeFile( clientPEM, diff --git a/packages/mongodb-runner/src/util.ts b/packages/mongodb-runner/src/util.ts index fcb1065a..c4c51bea 100644 --- a/packages/mongodb-runner/src/util.ts +++ b/packages/mongodb-runner/src/util.ts @@ -32,6 +32,30 @@ export async function parallelForEach( return await Promise.allSettled(result); } +/** + * A version of `Promise.all` that waits for all Promises to settle, + * and if any are rejected, throws an `AggregateError` with all errors. + * + * This has the benefit of allowing all Promises to complete, rather than + * failing fast on the first rejection (and potentially leaving more Promises + * in an unhandled rejection state). + */ +export async function safePromiseAll( + promises: (Promise | T)[], +): Promise { + const results = await Promise.allSettled(promises); + const rejected = results.filter( + (r): r is PromiseRejectedResult => r.status === 'rejected', + ); + if (rejected.length) { + throw new AggregateError( + [rejected.map((r) => r.reason)], + `${rejected.length} errors: ${rejected.map((r) => r.reason).join(', ')}`, + ); + } + return results.map((r) => (r as PromiseFulfilledResult).value); +} + export function pick( obj: T, keys: K[], diff --git a/packages/mongodb-runner/tsconfig.json b/packages/mongodb-runner/tsconfig.json index f029e4a0..16c4cb0f 100644 --- a/packages/mongodb-runner/tsconfig.json +++ b/packages/mongodb-runner/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "@mongodb-js/tsconfig-devtools/tsconfig.common.json", "compilerOptions": { - "target": "es2020", - "lib": ["es2020"] + "target": "es2021", + "lib": ["es2021"] } } From 83f8a1546b2c86023077032e481bf5d11339c40d Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 27 Jan 2026 23:17:26 +0100 Subject: [PATCH 19/21] fixup: increase windows timeout for cli tests --- packages/mongodb-runner/src/cli.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mongodb-runner/src/cli.spec.ts b/packages/mongodb-runner/src/cli.spec.ts index bbba1fa0..be2b3ff1 100644 --- a/packages/mongodb-runner/src/cli.spec.ts +++ b/packages/mongodb-runner/src/cli.spec.ts @@ -22,7 +22,7 @@ async function runCli( } describe('cli', function () { - this.timeout(100_000); + this.timeout(process.platform === 'win32' ? 400_000 : 100_000); let tmpDir = ''; before(async function () { From 6c70643b97119105845c5dfb0d156c1c24ae6a3a Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 28 Jan 2026 14:27:50 +0100 Subject: [PATCH 20/21] fixup: skip TLS client key generation for docker setups --- packages/mongodb-runner/src/mongocluster.ts | 4 +++- packages/mongodb-runner/src/tls-helpers.ts | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/mongodb-runner/src/mongocluster.ts b/packages/mongodb-runner/src/mongocluster.ts index df3658fd..533ec26b 100644 --- a/packages/mongodb-runner/src/mongocluster.ts +++ b/packages/mongodb-runner/src/mongocluster.ts @@ -543,7 +543,9 @@ export class MongoCluster extends EventEmitter { // Allow connection errors if automatic tls client key addition is disabled // since that option implies to users that they are taking responsibility for // ensuring that the correct runner instance is used for managing processes. - if (options.tlsAddClientKey !== false) throw err; + // Similarly, adding TLS client keys automatically in Docker may not be + // reliably possible due to filesystem path differences. + if (options.tlsAddClientKey !== false && !options.docker) throw err; } return cluster; } diff --git a/packages/mongodb-runner/src/tls-helpers.ts b/packages/mongodb-runner/src/tls-helpers.ts index fed5ae5d..3f1eab50 100644 --- a/packages/mongodb-runner/src/tls-helpers.ts +++ b/packages/mongodb-runner/src/tls-helpers.ts @@ -11,6 +11,7 @@ export interface TLSClientOptions { args?: string[]; tmpDir: string; internalClientOptions?: Partial; + docker?: unknown; } export async function handleTLSClientKeyOptions({ @@ -18,12 +19,13 @@ export async function handleTLSClientKeyOptions({ args: [...args] = [], tmpDir, internalClientOptions = {}, + docker, }: TLSClientOptions): Promise> { const existingTLSCAOptionIndex = args.findIndex((arg) => - arg.match(/^--tls(Cluster)?CAFile(=|$)/), + arg.match(/^--(tls|ssl)(Cluster)?CAFile(=|$)/), ); - if (tlsAddClientKey === false) return {}; + if (tlsAddClientKey === false || docker) return {}; if (tlsAddClientKey !== true && existingTLSCAOptionIndex === -1) return {}; if (tlsAddClientKey !== true && internalClientOptions.tlsCertificateKeyFile) return {}; From e8a3f1d2861dfcb20dc69beae8abb386acffc26b Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 29 Jan 2026 18:55:53 +0100 Subject: [PATCH 21/21] fixup: further simplify, add more tests ... --- .../mongodb-runner/src/mongocluster.spec.ts | 59 +++++++++++++++++ packages/mongodb-runner/src/mongoserver.ts | 64 +++++++++++++------ packages/mongodb-runner/src/tls-helpers.ts | 5 +- packages/mongodb-runner/src/util.ts | 1 + 4 files changed, 109 insertions(+), 20 deletions(-) diff --git a/packages/mongodb-runner/src/mongocluster.spec.ts b/packages/mongodb-runner/src/mongocluster.spec.ts index 7c4b9a82..d6dc855b 100644 --- a/packages/mongodb-runner/src/mongocluster.spec.ts +++ b/packages/mongodb-runner/src/mongocluster.spec.ts @@ -409,6 +409,65 @@ describe('MongoCluster', function () { await cluster.close(); }); + it('can serialize and deserialize sharded cluster with keyfile and auth (no enableTestCommands)', async function () { + const keyFile = path.join(tmpDir, 'keyFile2'); + await fs.writeFile(keyFile, 'secret', { mode: 0o400 }); + cluster = await MongoCluster.start({ + version: '8.x', + topology: 'sharded', + tmpDir, + secondaries: 0, + users: [ + { + username: 'testuser', + password: 'testpass', + roles: [{ role: 'readWriteAnyDatabase', db: 'admin' }], + }, + ], + args: ['--keyFile', keyFile], + }); + cluster = await MongoCluster.deserialize(cluster.serialize()); + await cluster.withClient(async (client) => { + expect( + (await client.db('admin').command({ connectionStatus: 1 })).authInfo + .authenticatedUsers, + ).to.deep.equal([{ user: 'testuser', db: 'admin' }]); + }); + await cluster.close(); + }); + + it('can serialize and deserialize sharded cluster with keyfile and auth (enableTestCommands=true)', async function () { + const keyFile = path.join(tmpDir, 'keyFile3'); + await fs.writeFile(keyFile, 'secret', { mode: 0o400 }); + cluster = await MongoCluster.start({ + version: '8.x', + topology: 'sharded', + tmpDir, + secondaries: 0, + users: [ + { + username: 'testuser', + password: 'testpass', + roles: [{ role: 'readWriteAnyDatabase', db: 'admin' }], + }, + ], + args: ['--keyFile', keyFile, '--setParameter', 'enableTestCommands=true'], + }); + cluster = await MongoCluster.deserialize(cluster.serialize()); + const doc = await cluster.withClient( + async (client) => { + expect( + (await client.db('admin').command({ connectionStatus: 1 })).authInfo + .authenticatedUsers, + ).to.deep.equal([{ user: '__system', db: 'local' }]); + return await client.db('config').collection('mongodbrunner').findOne(); + }, + { auth: { username: '__system', password: 'secret' } }, + ); + expect(doc?._id).to.be.a('string'); + await cluster.close(); + }); + it('can let callers listen for server log events', async function () { cluster = await MongoCluster.start({ version: '8.x', diff --git a/packages/mongodb-runner/src/mongoserver.ts b/packages/mongodb-runner/src/mongoserver.ts index 190c4087..a19da349 100644 --- a/packages/mongodb-runner/src/mongoserver.ts +++ b/packages/mongodb-runner/src/mongoserver.ts @@ -43,6 +43,10 @@ export interface MongoServerOptions { internalClientOptions?: Partial; /** Internal option -- if this is an arbiter, it does not understand user auth */ isArbiter?: boolean; + /** Internal option -- if this is a mongos instance */ + isMongos?: boolean; + /** Internal option -- if this is a configsvr instance */ + isConfigSvr?: boolean; /** Internal option -- if keyfile auth is used, this will be its contents */ keyFileContents?: string; } @@ -56,6 +60,8 @@ interface SerializedServerProperties { startTime: string; hasInsertedMetadataCollEntry: boolean; isArbiter?: boolean; + isMongos?: boolean; + isConfigSvr?: boolean; keyFileContents?: string; } @@ -84,6 +90,8 @@ export class MongoServer extends EventEmitter { private startTime = new Date().toISOString(); private hasInsertedMetadataCollEntry = false; public isArbiter = false; + public isMongos = false; + private isConfigSvr = false; private keyFileContents?: string; private defaultConnectionOptions?: Partial; @@ -106,6 +114,8 @@ export class MongoServer extends EventEmitter { hasInsertedMetadataCollEntry: this.hasInsertedMetadataCollEntry, defaultConnectionOptions: jsonClone(this.defaultConnectionOptions ?? {}), isArbiter: this.isArbiter, + isMongos: this.isMongos, + isConfigSvr: this.isConfigSvr, keyFileContents: this.keyFileContents, }; } @@ -119,6 +129,8 @@ export class MongoServer extends EventEmitter { srv.defaultConnectionOptions = serialized.defaultConnectionOptions; srv.closing = !!(await srv._populateBuildInfo('restore-check')); srv.isArbiter = !!serialized.isArbiter; + srv.isMongos = !!serialized.isMongos; + srv.isConfigSvr = !!serialized.isConfigSvr; srv.keyFileContents = serialized.keyFileContents; if (!srv.closing) { srv.pid = serialized.pid; @@ -190,6 +202,8 @@ export class MongoServer extends EventEmitter { const srv = new MongoServer(); srv.defaultConnectionOptions = { ...options.internalClientOptions }; srv.isArbiter = !!options.isArbiter; + srv.isMongos = options.binary === 'mongos'; + srv.isConfigSvr = !!options.args?.includes('--configsvr'); const keyFilePath = getKeyFileOption(options.args); if (keyFilePath) { srv.keyFileContents = await fs.readFile(keyFilePath, 'utf8'); @@ -340,21 +354,25 @@ export class MongoServer extends EventEmitter { async updateDefaultConnectionOptions( options: Partial, ): Promise { - let buildInfoError: Error | null = null; - for (let attempts = 0; attempts < 10; attempts++) { - buildInfoError = await this._populateBuildInfo('restore-check', { - ...options, - }); - if (!buildInfoError) break; - debug( - 'failed to get buildInfo when setting new options', - buildInfoError, - options, - this.connectionString, - ); - await sleep(2 ** attempts * 10); + const start = Date.now(); + let error: unknown; + for (let attempts = 0; attempts < 20; attempts++) { + try { + this.buildInfo = await this.withClient(async (client) => { + return await client.db('admin').command({ buildInfo: 1 }); + }, options); + error = undefined; + break; + } catch (err) { + error = err; + } + await sleep(Math.min(1000, 2 ** attempts * 10)); } - if (buildInfoError) throw buildInfoError; + debug('updated default connection options', { + error, + Δt: Date.now() - start, + }); + if (error) throw error; this.defaultConnectionOptions = { ...this.defaultConnectionOptions, ...options, @@ -421,7 +439,9 @@ export class MongoServer extends EventEmitter { // mongos hosts require a bit of special treatment because they do not have // local storage of their own, so we store the metadata in the config database, // which may be accessed by multiple mongos instances. - debug('ensuring metadata collection entry', insertedInfo, { isMongoS }); + debug('ensuring metadata collection entry', insertedInfo, { + isMongoS, + }); if (mode === 'insert-new') { const existingEntry = await runnerColl.findOne(); if (!isMongoS && existingEntry) { @@ -472,7 +492,10 @@ export class MongoServer extends EventEmitter { password: this.keyFileContents, }; clientOpts.authMechanism = 'SCRAM-SHA-256'; - clientOpts.authSource = 'local'; + // If enableTestCommands=true is set, we can use `admin` as the auth db on mongos + // instances, otherwise we won't be able to insert at all + // https://github.com/mongodb/mongo/blob/3faba053373cff4ff90ffce64114a595b9a70226/src/mongo/db/auth/sasl_mechanism_registry.cpp#L102-L108 + clientOpts.authSource = this.isMongos ? 'admin' : 'local'; } this.buildInfo = await this.withClient(async (client) => { // Insert the metadata entry, except if we're a freshly started mongos @@ -548,14 +571,19 @@ export class MongoServer extends EventEmitter { if (!this.hasInsertedMetadataCollEntry) { debug('populating metadata collection entry after initial setup'); const err = await this._populateBuildInfo('insert-new'); - if (err) throw err; + if (err && !this.isMongos && !this.isConfigSvr) throw err; } if (!this.buildInfo) { throw new Error( `Server buildInfo is not populated ${JSON.stringify(this.serialize())}`, ); } - if (!this.hasInsertedMetadataCollEntry && !this.isArbiter) { + if ( + !this.hasInsertedMetadataCollEntry && + !this.isArbiter && + !this.isMongos && + !this.isConfigSvr + ) { throw new Error( `Server has not inserted metadata collection entry ${JSON.stringify(this.serialize())}`, ); diff --git a/packages/mongodb-runner/src/tls-helpers.ts b/packages/mongodb-runner/src/tls-helpers.ts index 3f1eab50..e04a251f 100644 --- a/packages/mongodb-runner/src/tls-helpers.ts +++ b/packages/mongodb-runner/src/tls-helpers.ts @@ -11,7 +11,6 @@ export interface TLSClientOptions { args?: string[]; tmpDir: string; internalClientOptions?: Partial; - docker?: unknown; } export async function handleTLSClientKeyOptions({ @@ -20,7 +19,9 @@ export async function handleTLSClientKeyOptions({ tmpDir, internalClientOptions = {}, docker, -}: TLSClientOptions): Promise> { +}: TLSClientOptions & { docker?: unknown }): Promise< + Partial +> { const existingTLSCAOptionIndex = args.findIndex((arg) => arg.match(/^--(tls|ssl)(Cluster)?CAFile(=|$)/), ); diff --git a/packages/mongodb-runner/src/util.ts b/packages/mongodb-runner/src/util.ts index c4c51bea..eaafe9c1 100644 --- a/packages/mongodb-runner/src/util.ts +++ b/packages/mongodb-runner/src/util.ts @@ -48,6 +48,7 @@ export async function safePromiseAll( (r): r is PromiseRejectedResult => r.status === 'rejected', ); if (rejected.length) { + if (rejected.length === 1) throw rejected[0].reason; throw new AggregateError( [rejected.map((r) => r.reason)], `${rejected.length} errors: ${rejected.map((r) => r.reason).join(', ')}`,