Fix flaky integration tests and add test logging

.github/workflows/push.yml

@@ -15,12 +15,16 @@ jobs:
             matrix:
                 os: [windows-latest, ubuntu-latest]
                 node-version: [14.x, 18.x]
+            fail-fast: false
 
         defaults:
             run:
                 shell: bash
                 working-directory: packages/databricks-sdk-js
-
+        env:
+            DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+            DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+            TEST_DEFAULT_CLUSTER_ID: ${{ secrets.TEST_DEFAULT_CLUSTER_ID }}
         steps:
             - uses: actions/checkout@v3
 
@@ -41,12 +45,14 @@ jobs:
             - name: Unit Tests With Code Cov
               run: yarn test:cov
 
-            - name: Integration Tests With Code Cov
-              run: yarn run test:integ:cov
-              env:
-                  DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
-                  DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
-                  TEST_DEFAULT_CLUSTER_ID: ${{ secrets.TEST_DEFAULT_CLUSTER_ID }}
+            - name: Integration Tests
+              uses: nick-fields/retry@v2
+              with:
+                  max_attempts: 3
+                  retry_wait_seconds: 2
+                  timeout_minutes: 10
+                  retry_on: any
+                  command: yarn run test:integ:cov
 
             - run: |
                   cd coverage
@@ -66,6 +72,7 @@ jobs:
                 os: [windows-latest, macos-latest]
                 node-version: [16.x]
                 vscode-version: [stable, insiders]
+            fail-fast: false
 
         env:
             VSCODE_TEST_VERSION: ${{ matrix.vscode-version }}
@@ -104,6 +111,7 @@ jobs:
               working-directory: packages/databricks-vscode
 
             - name: Show coverage test result
+              continue-on-error: true
               run: yarn nyc report --reporter text -t ./coverage >> $GITHUB_STEP_SUMMARY
               working-directory: packages/databricks-vscode
 
@@ -112,12 +120,22 @@ jobs:
               with:
                   max_attempts: 3
                   retry_wait_seconds: 2
-                  timeout_minutes: 2
-                  retry_on: error
+                  timeout_minutes: 10
+                  retry_on: any
                   command: |
+                      yarn workspace databricks run test:integ:clean
+                      yarn install --immutable
+                      yarn build
                       cd packages/databricks-vscode
-                      yarn run test:integ:prepare
-                      yarn run test:integ:run
+                      yarn run test:integ
+
+            - name: Upload test logs
+              if: always()
+              continue-on-error: true
+              uses: actions/upload-artifact@v3
+              with:
+                  name: test-logs ${{ join(matrix.*, ' - ') }} - ${{ github.event_name }}
+                  path: packages/databricks-vscode/src/test/logs
 
     package:
         name: Package VSIX

.gitignore

@@ -12,4 +12,5 @@ out/
 !.yarn/sdks
 !.yarn/versions
 **/coverage/
-**/.nyc_output/
+**/.nyc_output/
+.databricks/**

packages/databricks-sdk-js/src/retries/Time.ts

@@ -17,22 +17,22 @@ export default class Time {
     }
 
     public toMillSeconds(): Time {
-        let secondsValue = 0;
+        let milliSecondsValue = 0;
         switch (this.units) {
             case TimeUnits.hours:
-                secondsValue = this.value * 60 * 60 * 1000;
+                milliSecondsValue = this.value * 60 * 60 * 1000;
                 break;
             case TimeUnits.minutes:
-                secondsValue = this.value * 60 * 1000;
+                milliSecondsValue = this.value * 60 * 1000;
                 break;
             case TimeUnits.seconds:
-                secondsValue = this.value * 1000;
+                milliSecondsValue = this.value * 1000;
                 break;
             case TimeUnits.milliseconds:
-                secondsValue = this.value;
+                milliSecondsValue = this.value;
                 break;
         }
-        return new Time(secondsValue, TimeUnits.seconds);
+        return new Time(milliSecondsValue, TimeUnits.milliseconds);
     }
 
     public add(other: Time): Time {

packages/databricks-vscode/.gitignore

@@ -1 +1,2 @@
+**/logs/**
 bin/**

packages/databricks-vscode/package.json

@@ -282,9 +282,11 @@
         "fix": "prettier . --write",
         "test:lint": "eslint src --ext ts && prettier . -c",
         "test:unit": "yarn run build && node ./out/test/runTest.js",
-        "test:integ:prepare": "yarn run package && extest get-vscode --code_version 1.69.1 --storage /tmp/vscode-test-databricks && extest get-chromedriver --storage /tmp/vscode-test-databricks && extest install-vsix -f databricks-0.0.1.vsix --storage /tmp/vscode-test-databricks",
-        "test:integ:run": "yarn run build && node out/test/e2e/scripts/e2e.js --storage /tmp/vscode-test-databricks --code_settings src/test/e2e/settings.json 'out/**/*.e2e.js'",
-        "test:integ": "yarn run test:integ:prepare && yarn run test:integ:run",
+        "test:integ:clean": "yarn run clean && rm -rf /tmp/vscode-test-databricks /tmp/databricks-vscode-test-extensions",
+        "test:integ:prepare": "yarn run package && extest get-vscode --code_version 1.69.1 --storage /tmp/vscode-test-databricks && extest get-chromedriver --storage /tmp/vscode-test-databricks",
+        "test:integ:install-vsix": "extest install-vsix --storage /tmp/vscode-test-databricks -f databricks-0.0.1.vsix -e /tmp/databricks-vscode-test-extensions",
+        "test:integ:run": "yarn run build && node out/test/e2e/scripts/e2e.js -e /tmp/databricks-vscode-test-extensions --storage /tmp/vscode-test-databricks --code_settings src/test/e2e/settings.json 'out/**/*.e2e.js'",
+        "test:integ": "yarn run test:integ:prepare && yarn run test:integ:install-vsix && yarn run test:integ:run",
         "test:cov": "nyc yarn run test:unit",
         "test": "yarn run test:lint && yarn run test:unit",
         "clean": "rm -rf node_modules out .vscode-test"
@@ -313,7 +315,8 @@
         "ts-mockito": "^2.6.1",
         "typescript": "^4.7.4",
         "vsce": "^2.10.0",
-        "vscode-extension-tester": "^4.4.1"
+        "vscode-extension-tester": "^4.4.1",
+        "winston": "^3.8.1"
     },
     "nyc": {
         "extends": "@istanbuljs/nyc-config-typescript",

packages/databricks-vscode/src/test/PeriodicRunner.ts

@@ -0,0 +1,40 @@
+import Time, {TimeUnits} from "@databricks/databricks-sdk/dist/retries/Time";
+
+export interface RunFunctionDetails {
+    fn: () => Promise<void>;
+    cleanup?: () => Promise<void>;
+    every: Time;
+    timer?: NodeJS.Timer;
+}
+
+export class PeriodicRunner {
+    private runFunctions: RunFunctionDetails[] = [];
+
+    runFunction(runFunctionDetails: RunFunctionDetails) {
+        this.runFunctions.push(runFunctionDetails);
+        return this;
+    }
+
+    start() {
+        this.runFunctions.forEach((runFunction, idx) => {
+            this.runFunctions[idx].timer = setInterval(
+                runFunction.fn,
+                runFunction.every.toMillSeconds().value
+            );
+        });
+    }
+
+    async stop() {
+        this.runFunctions.forEach((runFunction) => {
+            if (runFunction.timer) {
+                clearInterval(runFunction.timer);
+            }
+        });
+
+        this.runFunctions.forEach(async (runFunction) => {
+            if (runFunction.cleanup) {
+                await runFunction.cleanup();
+            }
+        });
+    }
+}

packages/databricks-vscode/src/test/e2e/configure.e2e.ts

@@ -1,5 +1,5 @@
 import assert from "node:assert";
-import path from "node:path";
+import path, {resolve} from "node:path";
 import * as fs from "fs/promises";
 import * as tmp from "tmp-promise";
 import {
@@ -12,8 +12,11 @@ import {
     CustomTreeSection,
 } from "vscode-extension-tester";
 import {getViewSection, openFolder, waitForTreeItems} from "./utils";
+import Time, {TimeUnits} from "@databricks/databricks-sdk/dist/retries/Time";
+import {PeriodicRunner} from "../PeriodicRunner";
+import {ImageLogger, Logger} from "../loggingUtils";
 
-describe("Configure Databricks Extension", function () {
+describe("Configure Databricks Extension", async function () {
     // these will be populated by the before() function
     let browser: VSBrowser;
     let driver: WebDriver;
@@ -22,10 +25,11 @@ describe("Configure Databricks Extension", function () {
 
     // this will be populated by the tests
     let clusterId: string;
+    let periodicRunners = new Map<string, PeriodicRunner>();
 
     this.timeout(10 * 60 * 1000);
 
-    before(async () => {
+    before(async function () {
         browser = VSBrowser.instance;
         driver = browser.driver;
 
@@ -36,16 +40,54 @@ describe("Configure Databricks Extension", function () {
         await openFolder(browser, projectDir);
     });
 
-    after(() => {
+    beforeEach(async function () {
+        const testName = this.currentTest?.title ?? "Default";
+        const suiteName = this.currentTest?.parent?.title ?? "Default";
+
+        const logger = await Logger.getLogger(suiteName, testName);
+        const imageLogger = ImageLogger.getLogger(suiteName, testName);
+
+        const periodicRunner = new PeriodicRunner()
+            .runFunction({
+                fn: async () => {
+                    (await driver.manage().logs().get("browser")).map(
+                        (entry) => {
+                            logger.info(entry.message);
+                        }
+                    );
+                },
+                every: new Time(1, TimeUnits.seconds),
+            })
+            .runFunction({
+                fn: async () => {
+                    await imageLogger.log(await driver.takeScreenshot());
+                },
+                cleanup: async () => {
+                    await imageLogger.flush();
+                },
+                every: new Time(1, TimeUnits.seconds),
+            });
+
+        periodicRunner.start();
+        periodicRunners.set(testName, periodicRunner);
+    });
+
+    afterEach(async function () {
+        const testName = this.currentTest?.title ?? "Default";
+        await periodicRunners.get(testName)?.stop();
+        periodicRunners.delete(testName);
+    });
+
+    after(function () {
         cleanup();
     });
 
-    it("should open VSCode", async () => {
+    it("should open VSCode", async function () {
         const title = await driver.getTitle();
         assert(title.indexOf("Get Started") >= 0);
     });
 
-    it("should open databricks panel and login", async () => {
+    it("should open databricks panel and login", async function () {
         const section = await getViewSection("Configuration");
         assert(section);
         const welcome = await section.findWelcomeContent();
@@ -61,14 +103,14 @@ describe("Configure Databricks Extension", function () {
         assert(await waitForTreeItems(section));
     });
 
-    it("should dismiss notifications", async () => {
+    it("should dismiss notifications", async function () {
         const notifications = await new Workbench().getNotifications();
         for (const n of notifications) {
             await n.dismiss();
         }
     });
 
-    it("shoult list clusters", async () => {
+    it("shoult list clusters", async function () {
         const section = await getViewSection("Clusters");
         assert(section);
         const tree = section as CustomTreeSection;
@@ -82,7 +124,7 @@ describe("Configure Databricks Extension", function () {
 
     // test is skipped because context menus currently don't work in vscode-extension-tester
     // https://github.com/redhat-developer/vscode-extension-tester/issues/444
-    it.skip("should filter clusters", async () => {
+    it.skip("should filter clusters", async function () {
         const section = await getViewSection("Clusters");
         assert(section);
         const action = await section!.getAction("Filter clusters ...");
@@ -97,7 +139,7 @@ describe("Configure Databricks Extension", function () {
         assert(items.length > 0);
     });
 
-    it("should attach cluster", async () => {
+    it("should attach cluster", async function () {
         const config = await getViewSection("Configuration");
         assert(config);
         const configTree = config as CustomTreeSection;
@@ -128,7 +170,6 @@ describe("Configure Databricks Extension", function () {
 
         await input.setText(clusterId);
         await input.confirm();
-        await input.selectQuickPick(0);
 
         // get cluster ID
         const clusterPropsItems = await clusterConfigItem.getChildren();
@@ -142,7 +183,7 @@ describe("Configure Databricks Extension", function () {
         assert(clusterId);
     });
 
-    it("should write the project config file", async () => {
+    it("should write the project config file", async function () {
         let projectConfig = JSON.parse(
             await fs.readFile(
                 path.join(projectDir, ".databricks", "project.json"),

packages/databricks-vscode/src/test/e2e/scripts/e2e.ts

@@ -22,11 +22,15 @@ async function main(args: string[]) {
     );
 
     const {path: configFile, cleanup} = await tmp.file();
+    let host = process.env["DATABRICKS_HOST"];
+    if (!host.startsWith("http")) {
+        host = `https://${host}`;
+    }
     try {
         await fs.writeFile(
             configFile,
             `[DEFAULT]
-host = ${process.env["DATABRICKS_HOST"]}
+host = ${host}
 token = ${process.env["DATABRICKS_TOKEN"]}`
         );