From 0bd503b4af98f11aee9790e4516ad822cd932ed1 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Mon, 30 Oct 2023 09:59:40 +0100 Subject: [PATCH] chore(e2e): retry ttfq test to work around the flake COMPASS-7374 (#5021) * chore(e2e): retry ttfq test to work around the flake * chore(e2e): clean up failed remote start * chore(e2e): check the cmd in addition to name; more logging * chore(e2e): fix typo Co-authored-by: Anna Henningsen --------- Co-authored-by: Anna Henningsen --- packages/compass-e2e-tests/helpers/compass.ts | 66 ++++++++++++++++++- .../tests/time-to-first-query.test.ts | 10 +++ .../compass/src/setup-hadron-distribution.ts | 12 +--- 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/packages/compass-e2e-tests/helpers/compass.ts b/packages/compass-e2e-tests/helpers/compass.ts index 0454850e52f..828ccfbc084 100644 --- a/packages/compass-e2e-tests/helpers/compass.ts +++ b/packages/compass-e2e-tests/helpers/compass.ts @@ -153,6 +153,7 @@ export class Compass { } async recordLogs(): Promise { + debug('Setting up renderer log listeners ...'); const puppeteerBrowser = await this.browser.getPuppeteer(); const pages = await puppeteerBrowser.pages(); const page = pages[0]; @@ -579,6 +580,9 @@ async function startCompass(opts: StartCompassOptions = {}): Promise { const maybeWrappedBinary = (await opts.wrapBinary?.(binary)) ?? binary; process.env.APP_ENV = 'webdriverio'; + // For webdriverio env we are changing appName so that keychain records do not + // overlap with anything else + process.env.HADRON_PRODUCT_NAME_OVERRIDE = 'MongoDB Compass WebdriverIO'; process.env.DEBUG = `${process.env.DEBUG ?? ''},mongodb-compass:main:logging`; process.env.MONGODB_COMPASS_TEST_LOG_DIR = path.join(LOG_PATH, 'app'); process.env.CHROME_LOG_FILE = chromedriverLogPath; @@ -616,7 +620,67 @@ async function startCompass(opts: StartCompassOptions = {}): Promise { debug('Starting compass via webdriverio with the following configuration:'); debug(JSON.stringify(options, null, 2)); - const browser = await remote(options); + let browser: CompassBrowser; + + try { + browser = await remote(options); + } catch (err) { + debug('Failed to start remote webdriver session', { + error: (err as Error).stack, + }); + // Sometimes when webdriver fails to start the remote session, we end up + // with a running app that hangs the test runner in CI causing the run to + // fail with idle timeout. We will try to clean up a potentially hanging app + // before rethrowing an error + + // ps-list is ESM-only in recent versions. + // eslint-disable-next-line @typescript-eslint/consistent-type-imports + const { default: psList }: typeof import('ps-list') = await eval( + `import('ps-list')` + ); + const processList = await psList(); + + const filteredProcesses = processList.filter((p) => { + return ( + p.ppid === process.pid && + (p.cmd?.startsWith(binary) || + /(MongoDB Compass|Electron|electron)/.test(p.name)) + ); + }); + + debug( + filteredProcesses.length === 0 + ? `Found no application running that need to be closed (following processes were spawned by this: ${processList + .filter((p) => { + return p.ppid === process.pid; + }) + .map((p) => { + return p.name; + }) + .join(', ')})` + : `Found following applications running: ${filteredProcesses + .map((p) => { + return p.name; + }) + .join(', ')}` + ); + + filteredProcesses.forEach((p) => { + try { + debug(`Killing process ${p.name} with PID ${p.pid}`); + if (process.platform === 'win32') { + crossSpawn.sync('taskkill', ['/PID', String(p.pid), '/F', '/T']); + } else { + process.kill(p.pid); + } + } catch (err) { + debug(`Failed to kill process ${p.name} with PID ${p.pid}`, { + error: (err as Error).stack, + }); + } + }); + throw err; + } const compass = new Compass(browser, { testPackagedApp, diff --git a/packages/compass-e2e-tests/tests/time-to-first-query.test.ts b/packages/compass-e2e-tests/tests/time-to-first-query.test.ts index 703f0c7468f..6122606a648 100644 --- a/packages/compass-e2e-tests/tests/time-to-first-query.test.ts +++ b/packages/compass-e2e-tests/tests/time-to-first-query.test.ts @@ -22,6 +22,16 @@ describe('Time to first query', function () { }); it('can open compass, connect to a database and run a query on a collection (never seen welcome)', async function () { + // Starting the application with the webdriver.io fails on the first run + // sometimes due to devtools / selenium server failing to start in time. + // While the root cause is unknown, it usually passes just fine on a re-run + // or next application start, so we are just retrying the test here to + // work around the flake. + // + // We re-run the whole test to make sure that the timings for the test run + // are not skewed by waiting for the application to restart multiple times. + this.retries(5); + // start compass inside the test so that the time is measured together compass = await beforeTests({ firstRun: true }); diff --git a/packages/compass/src/setup-hadron-distribution.ts b/packages/compass/src/setup-hadron-distribution.ts index 59c31ed8361..d38dc87de0e 100644 --- a/packages/compass/src/setup-hadron-distribution.ts +++ b/packages/compass/src/setup-hadron-distribution.ts @@ -35,15 +35,9 @@ if ( // Name and version are setup outside of Application and before anything else // so that if uncaught exception happens we already show correct name and // version - app.setName(process.env.HADRON_PRODUCT_NAME); - - // For webdriverio env we are changing appName so that keychain records do not - // overlap with anything else. Only appName should be changed for the - // webdriverio environment that is running tests, all relevant paths are - // configured from the test runner. - if (process.env.APP_ENV === 'webdriverio') { - app.setName(`${app.getName()} Webdriverio`); - } + app.setName( + process.env.HADRON_PRODUCT_NAME_OVERRIDE ?? process.env.HADRON_PRODUCT_NAME + ); // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error setVersion is not a public method