From f665b282047870b685a02ee8c975d8e8229bad21 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Wed, 11 Jun 2025 18:21:00 +0200 Subject: [PATCH 1/9] chore: introduce new extension test mode --- tests/extension/extensionTest.ts | 69 ++++++++++++++++++++++++++++ tests/extension/playwright.config.ts | 64 ++++++++++++++++++++++++++ tests/page/pageTest.ts | 3 ++ 3 files changed, 136 insertions(+) create mode 100644 tests/extension/extensionTest.ts create mode 100644 tests/extension/playwright.config.ts diff --git a/tests/extension/extensionTest.ts b/tests/extension/extensionTest.ts new file mode 100644 index 0000000000000..c19e58ad6df80 --- /dev/null +++ b/tests/extension/extensionTest.ts @@ -0,0 +1,69 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { baseTest } from '../config/baseTest'; +import { chromium } from 'playwright'; +import { expect, type PageTestFixtures, type PageWorkerFixtures } from '../page/pageTestApi'; +import type { TraceViewerFixtures } from '../config/traceViewerFixtures'; +import { traceViewerFixtures } from '../config/traceViewerFixtures'; +export { expect } from '@playwright/test'; +import http from 'node:http'; +import path from 'node:path'; +import { AddressInfo } from 'node:net'; + +export const extensionTest = baseTest.extend(traceViewerFixtures).extend({ + browserVersion: [({ browser }) => browser.version(), { scope: 'worker' }], + browserMajorVersion: [({ browserVersion }, use) => use(Number(browserVersion.split('.')[0])), { scope: 'worker' }], + isAndroid: [false, { scope: 'worker' }], + isElectron: [false, { scope: 'worker' }], + electronMajorVersion: [0, { scope: 'worker' }], + isWebView2: [false, { scope: 'worker' }], + isHeadlessShell: [false, { scope: 'worker' }], + + browser: [async ({ playwright }, use, testInfo) => { + const httpServer = http.createServer(); + await new Promise(resolve => httpServer.listen(0, resolve)); + const pathToExtension = path.join(__dirname, '../../../playwright-mcp/extension'); + const context = await chromium.launchPersistentContext('', { + args: [ + `--disable-extensions-except=${pathToExtension}`, + `--load-extension=${pathToExtension}`, + '--enable-features=AllowContentInitiatedDataUrlNavigations', + ], + channel: 'chromium', + }); + const { CDPBridgeServer } = await import('../../../playwright-mcp/src/cdp-relay.ts'); + const server = new CDPBridgeServer(httpServer); + const origin = `ws://localhost:${(httpServer.address() as AddressInfo).port}`; + await expect.poll(() => context?.serviceWorkers()).toHaveLength(1); + await context.pages()[0].goto(new URL('/popup.html', context.serviceWorkers()[0].url()).toString()); + await context.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).clear(); + await context.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).fill(`${origin}${server.EXTENSION_PATH}`); + await context.pages()[0].getByRole('button', { name: 'Share This Tab' }).click(); + await context.pages()[0].goto('about:blank'); + const browser = await playwright.chromium.connectOverCDP(`${origin}${server.CDP_PATH}`); + await use(browser); + httpServer.close(); + }, { scope: 'worker' }], + + context: async ({ browser }, use) => { + await use(browser.contexts()[0]); + }, + + page: async ({ browser }, use) => { + await use(browser.contexts()[0].pages()[0]); + } +}); diff --git a/tests/extension/playwright.config.ts b/tests/extension/playwright.config.ts new file mode 100644 index 0000000000000..628828cb6a088 --- /dev/null +++ b/tests/extension/playwright.config.ts @@ -0,0 +1,64 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { config as loadEnv } from 'dotenv'; +loadEnv({ path: path.join(__dirname, '..', '..', '.env') }); +process.env.PWTEST_UNDER_TEST = '1'; + +import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions } from '@playwright/test'; +import * as path from 'path'; + +process.env.PWPAGE_IMPL = 'extension'; + +const outputDir = path.join(__dirname, '..', '..', 'test-results'); +const testDir = path.join(__dirname, '..'); +const config: Config = { + testDir, + outputDir, + timeout: 30000, + globalTimeout: 5400000, + workers: process.env.CI ? 1 : undefined, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 3 : 0, + reporter: process.env.CI ? [ + ['dot'], + ['json', { outputFile: path.join(outputDir, 'report.json') }], + ['blob', { fileName: `${process.env.PWTEST_BOT_NAME}.zip` }], + ] : 'line', + projects: [], +}; + +const metadata = { + platform: process.platform, + headless: true, + browserName: 'extension', + channel: undefined, + mode: 'default', + video: false, +}; + +config.projects.push({ + name: 'extension', + // Share screenshots with chromium. + snapshotPathTemplate: '{testDir}/{testFileDir}/{testFileName}-snapshots/{arg}-chromium{ext}', + use: { + browserName: 'chromium', + }, + testDir: path.join(testDir, 'page'), + metadata, +}); + +export default config; diff --git a/tests/page/pageTest.ts b/tests/page/pageTest.ts index f6647563885f6..7553229703e88 100644 --- a/tests/page/pageTest.ts +++ b/tests/page/pageTest.ts @@ -21,6 +21,7 @@ import { androidTest } from '../android/androidTest'; import { browserTest } from '../config/browserTest'; import { electronTest } from '../electron/electronTest'; import { webView2Test } from '../webview2/webView2Test'; +import { extensionTest } from '../extension/extensionTest'; import type { PageTestFixtures, PageWorkerFixtures } from './pageTestApi'; import type { ServerFixtures, ServerWorkerOptions } from '../config/serverFixtures'; import { expect as baseExpect } from '@playwright/test'; @@ -34,6 +35,8 @@ if (process.env.PWPAGE_IMPL === 'electron') impl = electronTest; if (process.env.PWPAGE_IMPL === 'webview2') impl = webView2Test; +if (process.env.PWPAGE_IMPL === 'extension') + impl = extensionTest; export const test = impl; From 0b336aee2c1053a5f2ccd636321327a69804d6c5 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Fri, 13 Jun 2025 01:10:24 +0200 Subject: [PATCH 2/9] nit --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 6171609f6cdca..7e287cfa88796 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "atest": "playwright test --config=tests/android/playwright.config.ts", "etest": "playwright test --config=tests/electron/playwright.config.ts", "webview2test": "playwright test --config=tests/webview2/playwright.config.ts", + "extensiontest": "playwright test --config=tests/extension/playwright.config.ts", "itest": "playwright test --config=tests/installation/playwright.config.ts", "stest": "playwright test --config=tests/stress/playwright.config.ts", "biditest": "playwright test --config=tests/bidi/playwright.config.ts", From 72e3c031b645128c9a95805faf001ebe64d64df1 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 12 Jun 2025 17:48:27 -0700 Subject: [PATCH 3/9] dialog fix --- tests/extension/extensionTest.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/extension/extensionTest.ts b/tests/extension/extensionTest.ts index c19e58ad6df80..05eb56a7dfbbf 100644 --- a/tests/extension/extensionTest.ts +++ b/tests/extension/extensionTest.ts @@ -55,6 +55,9 @@ export const extensionTest = baseTest.extend(traceViewerFix await context.pages()[0].getByRole('button', { name: 'Share This Tab' }).click(); await context.pages()[0].goto('about:blank'); const browser = await playwright.chromium.connectOverCDP(`${origin}${server.CDP_PATH}`); + context.on('dialog', dialog => { + // Make sure the dialog is not dismissed automatically. + }); await use(browser); httpServer.close(); }, { scope: 'worker' }], From 91ceaa142df5944bf552f9c78250f71028324a98 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Fri, 13 Jun 2025 10:17:20 +0200 Subject: [PATCH 4/9] test: fix worker tests (browserVersion fixture) --- tests/extension/extensionTest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/extension/extensionTest.ts b/tests/extension/extensionTest.ts index 05eb56a7dfbbf..09f18e878bb7f 100644 --- a/tests/extension/extensionTest.ts +++ b/tests/extension/extensionTest.ts @@ -25,7 +25,7 @@ import path from 'node:path'; import { AddressInfo } from 'node:net'; export const extensionTest = baseTest.extend(traceViewerFixtures).extend({ - browserVersion: [({ browser }) => browser.version(), { scope: 'worker' }], + browserVersion: [({ browser }, use) => use(browser.version()), { scope: 'worker' }], browserMajorVersion: [({ browserVersion }, use) => use(Number(browserVersion.split('.')[0])), { scope: 'worker' }], isAndroid: [false, { scope: 'worker' }], isElectron: [false, { scope: 'worker' }], From ea3abc15a80d1b31e3784a30eb52c820c5ab447e Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 13 Jun 2025 16:32:23 -0700 Subject: [PATCH 5/9] Update imports from playwright-mcp after recent renames --- tests/extension/extensionTest.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/extension/extensionTest.ts b/tests/extension/extensionTest.ts index 09f18e878bb7f..6cb3cf5e5bc87 100644 --- a/tests/extension/extensionTest.ts +++ b/tests/extension/extensionTest.ts @@ -45,16 +45,16 @@ export const extensionTest = baseTest.extend(traceViewerFix ], channel: 'chromium', }); - const { CDPBridgeServer } = await import('../../../playwright-mcp/src/cdp-relay.ts'); - const server = new CDPBridgeServer(httpServer); + const { CDPRelayServer } = await import('../../../playwright-mcp/src/cdpRelay.ts'); + new CDPRelayServer(httpServer); const origin = `ws://localhost:${(httpServer.address() as AddressInfo).port}`; await expect.poll(() => context?.serviceWorkers()).toHaveLength(1); await context.pages()[0].goto(new URL('/popup.html', context.serviceWorkers()[0].url()).toString()); await context.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).clear(); - await context.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).fill(`${origin}${server.EXTENSION_PATH}`); + await context.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).fill(`${origin}/extension`); await context.pages()[0].getByRole('button', { name: 'Share This Tab' }).click(); await context.pages()[0].goto('about:blank'); - const browser = await playwright.chromium.connectOverCDP(`${origin}${server.CDP_PATH}`); + const browser = await playwright.chromium.connectOverCDP(`${origin}/cdp`); context.on('dialog', dialog => { // Make sure the dialog is not dismissed automatically. }); From c1fcdccc25c79ed4f5bb36f7705c9d2000e891e4 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Thu, 19 Jun 2025 22:59:53 +0200 Subject: [PATCH 6/9] allow using CRPATH --- tests/extension/extensionTest.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/extension/extensionTest.ts b/tests/extension/extensionTest.ts index 6cb3cf5e5bc87..f982bb6bb16d0 100644 --- a/tests/extension/extensionTest.ts +++ b/tests/extension/extensionTest.ts @@ -38,6 +38,7 @@ export const extensionTest = baseTest.extend(traceViewerFix await new Promise(resolve => httpServer.listen(0, resolve)); const pathToExtension = path.join(__dirname, '../../../playwright-mcp/extension'); const context = await chromium.launchPersistentContext('', { + executablePath: process.env.CRPATH, args: [ `--disable-extensions-except=${pathToExtension}`, `--load-extension=${pathToExtension}`, From bf2ca9bbfbfd8e86d30b6d02156a99c68fbec92d Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 19 Jun 2025 16:55:58 -0700 Subject: [PATCH 7/9] Create new page (and new CDP connection) for every test --- tests/extension/extensionTest.ts | 62 +++++++++++++++++++++------- tests/extension/playwright.config.ts | 1 + 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/tests/extension/extensionTest.ts b/tests/extension/extensionTest.ts index f982bb6bb16d0..139e0b7d98a8e 100644 --- a/tests/extension/extensionTest.ts +++ b/tests/extension/extensionTest.ts @@ -15,7 +15,7 @@ */ import { baseTest } from '../config/baseTest'; -import { chromium } from 'playwright'; +import { chromium, type BrowserContext } from 'playwright'; import { expect, type PageTestFixtures, type PageWorkerFixtures } from '../page/pageTestApi'; import type { TraceViewerFixtures } from '../config/traceViewerFixtures'; import { traceViewerFixtures } from '../config/traceViewerFixtures'; @@ -24,7 +24,13 @@ import http from 'node:http'; import path from 'node:path'; import { AddressInfo } from 'node:net'; -export const extensionTest = baseTest.extend(traceViewerFixtures).extend({ +export type ExtensionTestFixtures = { + persistentContext: BrowserContext; + relayServer: http.Server; +}; + + +export const extensionTest = baseTest.extend(traceViewerFixtures).extend({ browserVersion: [({ browser }, use) => use(browser.version()), { scope: 'worker' }], browserMajorVersion: [({ browserVersion }, use) => use(Number(browserVersion.split('.')[0])), { scope: 'worker' }], isAndroid: [false, { scope: 'worker' }], @@ -33,9 +39,16 @@ export const extensionTest = baseTest.extend(traceViewerFix isWebView2: [false, { scope: 'worker' }], isHeadlessShell: [false, { scope: 'worker' }], - browser: [async ({ playwright }, use, testInfo) => { + relayServer: [async ({ }, use) => { const httpServer = http.createServer(); await new Promise(resolve => httpServer.listen(0, resolve)); + const { CDPRelayServer } = await import('../../../playwright-mcp/src/cdpRelay.ts'); + new CDPRelayServer(httpServer); + await use(httpServer); + httpServer.close(); + }, { scope: 'worker' }], + + persistentContext: [async ({ }, use) => { const pathToExtension = path.join(__dirname, '../../../playwright-mcp/extension'); const context = await chromium.launchPersistentContext('', { executablePath: process.env.CRPATH, @@ -46,28 +59,45 @@ export const extensionTest = baseTest.extend(traceViewerFix ], channel: 'chromium', }); - const { CDPRelayServer } = await import('../../../playwright-mcp/src/cdpRelay.ts'); - new CDPRelayServer(httpServer); - const origin = `ws://localhost:${(httpServer.address() as AddressInfo).port}`; - await expect.poll(() => context?.serviceWorkers()).toHaveLength(1); - await context.pages()[0].goto(new URL('/popup.html', context.serviceWorkers()[0].url()).toString()); - await context.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).clear(); - await context.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).fill(`${origin}/extension`); - await context.pages()[0].getByRole('button', { name: 'Share This Tab' }).click(); - await context.pages()[0].goto('about:blank'); - const browser = await playwright.chromium.connectOverCDP(`${origin}/cdp`); context.on('dialog', dialog => { // Make sure the dialog is not dismissed automatically. }); + await use(context); + await context.close(); + }, { scope: 'worker' }], + + browser: [async ({ persistentContext, relayServer, playwright }, use, testInfo) => { + const origin = `ws://localhost:${(relayServer.address() as AddressInfo).port}`; + await expect.poll(() => persistentContext.serviceWorkers()).toHaveLength(1); + await persistentContext.pages()[0].goto(new URL('/popup.html', persistentContext.serviceWorkers()[0].url()).toString()); + await persistentContext.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).clear(); + await persistentContext.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).fill(`${origin}/extension`); + await persistentContext.pages()[0].getByRole('button', { name: 'Share This Tab' }).click(); + await persistentContext.pages()[0].goto('about:blank'); + const browser = await playwright.chromium.connectOverCDP(`${origin}/cdp`); await use(browser); - httpServer.close(); }, { scope: 'worker' }], context: async ({ browser }, use) => { await use(browser.contexts()[0]); }, - page: async ({ browser }, use) => { - await use(browser.contexts()[0].pages()[0]); + page: async ({ persistentContext, relayServer, playwright }, use) => { + const page = await persistentContext.newPage(); + const origin = `ws://localhost:${(relayServer.address() as AddressInfo).port}`; + await expect.poll(() => persistentContext.serviceWorkers()).toHaveLength(1); + await page.goto(new URL('/popup.html', persistentContext.serviceWorkers()[0].url()).toString()); + await page.getByRole('textbox', { name: 'Bridge Server URL:' }).clear(); + await page.getByRole('textbox', { name: 'Bridge Server URL:' }).fill(`${origin}/extension`); + await page.getByRole('button', { name: 'Share This Tab' }).click(); + await page.goto('about:blank'); + const browser = await playwright.chromium.connectOverCDP(`${origin}/cdp`); + const pages = browser.contexts()[0].pages(); + const remotePage = pages[pages.length - 1]; + await use(remotePage); + // Disconnect from the tab. + await browser.close(); + // Close the page. + await page.close(); } }); diff --git a/tests/extension/playwright.config.ts b/tests/extension/playwright.config.ts index 628828cb6a088..3bee3341a2e1c 100644 --- a/tests/extension/playwright.config.ts +++ b/tests/extension/playwright.config.ts @@ -31,6 +31,7 @@ const config: Config = { timeout: 30000, globalTimeout: 5400000, workers: process.env.CI ? 1 : undefined, + fullyParallel: !process.env.CI, forbidOnly: !!process.env.CI, retries: process.env.CI ? 3 : 0, reporter: process.env.CI ? [ From 65d9969af3fb98eb03016d277907d2982e39335e Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Fri, 20 Jun 2025 11:17:16 -0700 Subject: [PATCH 8/9] Throw from browser and context fixtures --- tests/extension/extensionTest.ts | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/tests/extension/extensionTest.ts b/tests/extension/extensionTest.ts index 139e0b7d98a8e..1fecbbe367116 100644 --- a/tests/extension/extensionTest.ts +++ b/tests/extension/extensionTest.ts @@ -31,7 +31,7 @@ export type ExtensionTestFixtures = { export const extensionTest = baseTest.extend(traceViewerFixtures).extend({ - browserVersion: [({ browser }, use) => use(browser.version()), { scope: 'worker' }], + browserVersion: [({ persistentContext }, use) => use(persistentContext.browser().version()), { scope: 'worker' }], browserMajorVersion: [({ browserVersion }, use) => use(Number(browserVersion.split('.')[0])), { scope: 'worker' }], isAndroid: [false, { scope: 'worker' }], isElectron: [false, { scope: 'worker' }], @@ -66,23 +66,15 @@ export const extensionTest = baseTest.extend(traceViewerFix await context.close(); }, { scope: 'worker' }], - browser: [async ({ persistentContext, relayServer, playwright }, use, testInfo) => { - const origin = `ws://localhost:${(relayServer.address() as AddressInfo).port}`; - await expect.poll(() => persistentContext.serviceWorkers()).toHaveLength(1); - await persistentContext.pages()[0].goto(new URL('/popup.html', persistentContext.serviceWorkers()[0].url()).toString()); - await persistentContext.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).clear(); - await persistentContext.pages()[0].getByRole('textbox', { name: 'Bridge Server URL:' }).fill(`${origin}/extension`); - await persistentContext.pages()[0].getByRole('button', { name: 'Share This Tab' }).click(); - await persistentContext.pages()[0].goto('about:blank'); - const browser = await playwright.chromium.connectOverCDP(`${origin}/cdp`); - await use(browser); + browser: [async ({ }, use) => { + throw new Error('Not supported in the extension tests'); }, { scope: 'worker' }], - context: async ({ browser }, use) => { - await use(browser.contexts()[0]); + context: async ({ }, use) => { + throw new Error('Not supported in the extension tests'); }, - page: async ({ persistentContext, relayServer, playwright }, use) => { + page: async ({ persistentContext, relayServer, playwright, server }, use) => { const page = await persistentContext.newPage(); const origin = `ws://localhost:${(relayServer.address() as AddressInfo).port}`; await expect.poll(() => persistentContext.serviceWorkers()).toHaveLength(1); From 2bbe24482302c93d5c3371f4141164929673250d Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Mon, 23 Jun 2025 09:51:35 +0200 Subject: [PATCH 9/9] change to chromium-tip-of-tree --- tests/extension/extensionTest.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/extension/extensionTest.ts b/tests/extension/extensionTest.ts index 1fecbbe367116..b1acd7c14d16f 100644 --- a/tests/extension/extensionTest.ts +++ b/tests/extension/extensionTest.ts @@ -57,7 +57,8 @@ export const extensionTest = baseTest.extend(traceViewerFix `--load-extension=${pathToExtension}`, '--enable-features=AllowContentInitiatedDataUrlNavigations', ], - channel: 'chromium', + // Depends on http://crrev.com/c/6639022. + channel: 'chromium-tip-of-tree', }); context.on('dialog', dialog => { // Make sure the dialog is not dismissed automatically.