chore(e2e): retry ttfq test to work around the flake COMPASS-7374 (#5021

) * chore(e2e): retry ttfq test to work around the flake * chore(e2e): clean up failed remote start * chore(e2e): check the cmd in addition to name; more logging * chore(e2e): fix typo Co-authored-by: Anna Henningsen <[email protected]> --------- Co-authored-by: Anna Henningsen <[email protected]>
mongodb-js · Oct 30, 2023 · 0bd503b · 0bd503b
1 parent 276de73
commit 0bd503b
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 10 deletions.
diff --git a/packages/compass-e2e-tests/helpers/compass.ts b/packages/compass-e2e-tests/helpers/compass.ts
@@ -153,6 +153,7 @@ export class Compass {
   }
 
   async recordLogs(): Promise<void> {
+    debug('Setting up renderer log listeners ...');
     const puppeteerBrowser = await this.browser.getPuppeteer();
     const pages = await puppeteerBrowser.pages();
     const page = pages[0];
@@ -579,6 +580,9 @@ async function startCompass(opts: StartCompassOptions = {}): Promise<Compass> {
   const maybeWrappedBinary = (await opts.wrapBinary?.(binary)) ?? binary;
 
   process.env.APP_ENV = 'webdriverio';
+  // For webdriverio env we are changing appName so that keychain records do not
+  // overlap with anything else
+  process.env.HADRON_PRODUCT_NAME_OVERRIDE = 'MongoDB Compass WebdriverIO';
   process.env.DEBUG = `${process.env.DEBUG ?? ''},mongodb-compass:main:logging`;
   process.env.MONGODB_COMPASS_TEST_LOG_DIR = path.join(LOG_PATH, 'app');
   process.env.CHROME_LOG_FILE = chromedriverLogPath;
@@ -616,7 +620,67 @@ async function startCompass(opts: StartCompassOptions = {}): Promise<Compass> {
   debug('Starting compass via webdriverio with the following configuration:');
   debug(JSON.stringify(options, null, 2));
 
-  const browser = await remote(options);
+  let browser: CompassBrowser;
+
+  try {
+    browser = await remote(options);
+  } catch (err) {
+    debug('Failed to start remote webdriver session', {
+      error: (err as Error).stack,
+    });
+    // Sometimes when webdriver fails to start the remote session, we end up
+    // with a running app that hangs the test runner in CI causing the run to
+    // fail with idle timeout. We will try to clean up a potentially hanging app
+    // before rethrowing an error
+
+    // ps-list is ESM-only in recent versions.
+    // eslint-disable-next-line @typescript-eslint/consistent-type-imports
+    const { default: psList }: typeof import('ps-list') = await eval(
+      `import('ps-list')`
+    );
+    const processList = await psList();
+
+    const filteredProcesses = processList.filter((p) => {
+      return (
+        p.ppid === process.pid &&
+        (p.cmd?.startsWith(binary) ||
+          /(MongoDB Compass|Electron|electron)/.test(p.name))
+      );
+    });
+
+    debug(
+      filteredProcesses.length === 0
+        ? `Found no application running that need to be closed (following processes were spawned by this: ${processList
+            .filter((p) => {
+              return p.ppid === process.pid;
+            })
+            .map((p) => {
+              return p.name;
+            })
+            .join(', ')})`
+        : `Found following applications running: ${filteredProcesses
+            .map((p) => {
+              return p.name;
+            })
+            .join(', ')}`
+    );
+
+    filteredProcesses.forEach((p) => {
+      try {
+        debug(`Killing process ${p.name} with PID ${p.pid}`);
+        if (process.platform === 'win32') {
+          crossSpawn.sync('taskkill', ['/PID', String(p.pid), '/F', '/T']);
+        } else {
+          process.kill(p.pid);
+        }
+      } catch (err) {
+        debug(`Failed to kill process ${p.name} with PID ${p.pid}`, {
+          error: (err as Error).stack,
+        });
+      }
+    });
+    throw err;
+  }
 
   const compass = new Compass(browser, {
     testPackagedApp,

diff --git a/packages/compass-e2e-tests/tests/time-to-first-query.test.ts b/packages/compass-e2e-tests/tests/time-to-first-query.test.ts
@@ -22,6 +22,16 @@ describe('Time to first query', function () {
   });
 
   it('can open compass, connect to a database and run a query on a collection (never seen welcome)', async function () {
+    // Starting the application with the webdriver.io fails on the first run
+    // sometimes due to devtools / selenium server failing to start in time.
+    // While the root cause is unknown, it usually passes just fine on a re-run
+    // or next application start, so we are just retrying the test here to
+    // work around the flake.
+    //
+    // We re-run the whole test to make sure that the timings for the test run
+    // are not skewed by waiting for the application to restart multiple times.
+    this.retries(5);
+
     // start compass inside the test so that the time is measured together
     compass = await beforeTests({ firstRun: true });
 

diff --git a/packages/compass/src/setup-hadron-distribution.ts b/packages/compass/src/setup-hadron-distribution.ts
@@ -35,15 +35,9 @@ if (
   // Name and version are setup outside of Application and before anything else
   // so that if uncaught exception happens we already show correct name and
   // version
-  app.setName(process.env.HADRON_PRODUCT_NAME);
-
-  // For webdriverio env we are changing appName so that keychain records do not
-  // overlap with anything else. Only appName should be changed for the
-  // webdriverio environment that is running tests, all relevant paths are
-  // configured from the test runner.
-  if (process.env.APP_ENV === 'webdriverio') {
-    app.setName(`${app.getName()} Webdriverio`);
-  }
+  app.setName(
+    process.env.HADRON_PRODUCT_NAME_OVERRIDE ?? process.env.HADRON_PRODUCT_NAME
+  );
 
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
   // @ts-expect-error setVersion is not a public method