[DECO-102] Setup logging infra (#117)

Example request response logs
```json
[
  {
    "level": "debug",
    "message": "REDACTED/api/2.0/preview/scim/v2/Me",
    "operationId": "b376e4c8-9140-49f5-873f-53dc4311c626",
    "operationName": "ApiClient.request",
    "request": {
      "agent": {
        "_events": {},
        "_eventsCount": 2,
        "_sessionCache": { "list": [], "map": {} },
        "defaultPort": 443,
        "freeSockets": {},
        "keepAlive": true,
        "keepAliveMsecs": 15000,
        "maxCachedSessions": 100,
        "maxFreeSockets": 256,
        "maxSockets": null,
        "maxTotalSockets": null,
        "options": { "keepAlive": true, "keepAliveMsecs": 15000, "path": null },
        "protocol": "https:",
        "requests": {},
        "scheduling": "lifo",
        "sockets": {},
        "totalSocketCount": 0
      },
      "headers": {
        "Authorization": "Bearer REDACTED",
        "Content-Type": "text/json",
        "User-Agent": "vscode-extension/0.0.1 databricks-sdk-js/0.0.1 nodejs/16.14.2 os/darwin"
      },
      "method": "GET"
    },
    "timestamp": "1664462278553"
  },
  
  {
    "level": "debug",
    "message": "REDACTED/api/2.0/preview/scim/v2/Me",
    "operationId": "b376e4c8-9140-49f5-873f-53dc4311c626",
    "operationName": "ApiClient.request",
    "response": {
      "active": true,
      "emails": [
        {
          "primary": true,
          "type": "work",
          "value": "serge.smertin@databricks.com"
        }
      ],
      "externalId": "471d1718-d938-4b13-95b8-fd1d4e48894b",
      "groups": [
        {
          "$ref": "Groups/737310559950744",
          "display": "admins",
          "type": "direct",
          "value": "737310559950744"
        }
      ],
      "id": "4183391249163402",
      "schemas": [
        "urn:ietf:params:scim:schemas:core:2.0:User",
        "urn:ietf:params:scim:schemas:extension:workspace:2.0:User:permissionLevel"
      ],
      "userName": "serge.smertin@databricks.com"
    },
    "timestamp": "1664462279368"
  }
]

```

Author: kartikgupta-db

package.json

@@ -16,7 +16,8 @@
         "test": "yarn workspaces foreach run test",
         "test:integ": "yarn workspaces foreach run test:integ",
         "build": "yarn workspaces foreach run build",
-        "clean": "yarn workspaces foreach run clean"
+        "clean": "yarn workspaces foreach run clean",
+        "fix": "yarn workspaces foreach run fix"
     },
     "repository": {
         "type": "git",

packages/databricks-sdk-js/package.json

@@ -30,7 +30,9 @@
     },
     "dependencies": {
         "ini": "^3.0.1",
-        "node-fetch": "^3.2.10"
+        "inversify": "^6.0.1",
+        "node-fetch": "^3.2.10",
+        "reflect-metadata": "^0.1.13"
     },
     "devDependencies": {
         "@istanbuljs/nyc-config-typescript": "^1.0.2",
@@ -39,7 +41,7 @@
         "@types/chai-spies": "^1.0.3",
         "@types/ini": "^1.3.31",
         "@types/mocha": "^9.1.1",
-        "@types/node": "^18.7.17",
+        "@types/node": "^18.8.2",
         "@types/tmp": "^0.2.3",
         "@types/uuid": "^8.3.4",
         "chai": "^4.3.6",

packages/databricks-sdk-js/src/Redactor.test.ts

@@ -0,0 +1,35 @@
+import assert from "assert";
+import {defaultRedactor} from ".";
+import {onlyNBytes} from "./Redactor";
+
+describe(__filename, () => {
+    it("should redact by field names", () => {
+        const testObj = {
+            prop: "value1",
+            nested: {
+                prop: false,
+                prop2: "value",
+                headers: {
+                    header1: "test",
+                    header2: "value",
+                },
+            },
+        };
+        defaultRedactor.addFieldName("prop");
+        const actual = defaultRedactor.sanitize(testObj, ["headers"]);
+        const expected = {
+            prop: "***REDACTED***",
+            nested: {
+                prop: "***REDACTED***",
+                prop2: "value",
+            },
+        };
+        assert.deepEqual(actual, expected);
+    });
+
+    it("should truncate string to n bytes", () => {
+        const n = 5;
+        const str = "1234567890";
+        assert.equal(onlyNBytes(str, n), "12345...(5 more bytes)");
+    });
+});

packages/databricks-sdk-js/src/Redactor.ts

@@ -0,0 +1,60 @@
+export function onlyNBytes(str: string, numBytes: number) {
+    return str.length > numBytes
+        ? str.slice(0, numBytes) + `...(${str.length - numBytes} more bytes)`
+        : str;
+}
+
+function isPrimitveType(obj: any) {
+    return Object(obj) !== obj;
+}
+
+export class Redactor {
+    constructor(private fieldNames: string[] = []) {}
+
+    addFieldName(fieldName: string) {
+        this.fieldNames.push(fieldName);
+    }
+
+    sanitize(
+        obj: any,
+        dropFields: string[] = [],
+        maxFieldLength: number = 96
+    ): any {
+        if (isPrimitveType(obj)) {
+            if (typeof obj === "string") {
+                return onlyNBytes(obj, maxFieldLength);
+            }
+            if (obj instanceof String) {
+                return onlyNBytes(obj.toString(), maxFieldLength);
+            }
+            return obj;
+        }
+
+        if (Array.isArray(obj)) {
+            return obj.map((e) => this.sanitize(e, dropFields, maxFieldLength));
+        }
+
+        //make a copy of the object
+        obj = JSON.parse(JSON.stringify(obj));
+        for (let key in obj) {
+            if (dropFields.includes(key)) {
+                delete obj[key];
+            } else if (
+                isPrimitveType(obj[key]) &&
+                this.fieldNames.includes(key)
+            ) {
+                obj[key] = "***REDACTED***";
+            } else {
+                obj[key] = this.sanitize(obj[key], dropFields, maxFieldLength);
+            }
+        }
+
+        return obj;
+    }
+}
+
+export const defaultRedactor = new Redactor([
+    "string_value",
+    "token_value",
+    "content",
+]);

packages/databricks-sdk-js/src/api-client.test.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-
+import ".";
 import assert from "node:assert";
 import {ApiClient} from "./api-client";
 

packages/databricks-sdk-js/src/api-client.ts

@@ -3,6 +3,7 @@ import * as https from "node:https";
 import {TextDecoder} from "node:util";
 import {fromDefaultChain} from "./auth/fromChain";
 import {fetch} from "./fetch";
+import {NamedLogger, loggerInstance, logOpId, withLogContext} from "./logging";
 import {CancellationToken} from "./types";
 
 const sdkVersion = require("../package.json").version;
@@ -51,11 +52,14 @@ export class ApiClient {
         return pairs.join(" ");
     }
 
+    @withLogContext("SDK")
     async request(
         path: string,
         method: HttpMethod,
         payload?: any,
-        cancellationToken?: CancellationToken
+        cancellationToken?: CancellationToken,
+        @logOpId() opId?: string,
+        @loggerInstance() logger?: NamedLogger
     ): Promise<Object> {
         const credentials = await this.credentialProvider();
         const headers = {
@@ -84,6 +88,7 @@ export class ApiClient {
         let response;
 
         try {
+            logger?.debug(url.toString(), {request: options});
             const {abort, response: responsePromise} = await fetch(
                 url.toString(),
                 options
@@ -119,6 +124,7 @@ export class ApiClient {
 
         try {
             response = JSON.parse(responseText);
+            logger?.debug(url.toString(), {response: response});
         } catch (e) {
             throw new ApiClientResponseError(responseText, response);
         }

packages/databricks-sdk-js/src/auth/configFile.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 import {readFile, stat} from "node:fs/promises";
 import {parse} from "ini";
 import {homedir} from "node:os";
+import {defaultRedactor} from "../Redactor";
 
 export type Profiles = Record<
     string,
@@ -25,6 +26,13 @@ export function resolveConfigFilePath(filePath?: string): string {
     return filePath;
 }
 
+function getProfile(config: any) {
+    return {
+        host: new URL(config.host),
+        token: config.token,
+    };
+}
+
 export async function loadConfigFile(filePath?: string): Promise<Profiles> {
     filePath = resolveConfigFilePath(filePath);
 
@@ -58,16 +66,10 @@ export async function loadConfigFile(filePath?: string): Promise<Profiles> {
                 defaultSectionFound = true;
                 continue;
             }
-            profiles[key] = {
-                host: new URL(config[key].host),
-                token: config[key].token,
-            };
+            profiles[key] = getProfile(config[key]);
         }
         if (defaultSectionFound) {
-            profiles["DEFAULT"] = {
-                host: new URL(defaultSection.host),
-                token: defaultSection.token,
-            };
+            profiles["DEFAULT"] = getProfile(defaultSection);
         }
     } catch (e: unknown) {
         let message;

packages/databricks-sdk-js/src/decorators.test.ts

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-
+import ".";
 import assert from "node:assert";
 import {ListRequest, ListReposResponse} from "./apis/repos";
 import {paginated} from "./decorators";

packages/databricks-sdk-js/src/index.ts

@@ -1,3 +1,5 @@
+import "reflect-metadata";
+
 export * from "./api-client";
 
 export * as cluster from "./apis/clusters";
@@ -40,3 +42,7 @@ export {ClusterFixture, TokenFixture} from "./test/fixtures";
 
 export {RetryConfigs, default as retry} from "./retries/retries";
 export {TimeUnits, default as Time} from "./retries/Time";
+
+export * as logging from "./logging";
+
+export {Redactor, defaultRedactor} from "./Redactor";

packages/databricks-sdk-js/src/logging/DefaultLogger.ts

@@ -0,0 +1,19 @@
+import {Writable} from "stream";
+import {LogEntry, Logger} from "./types";
+
+export class DefaultLogger implements Logger {
+    private _stream: Writable;
+    constructor(outputStream?: Writable) {
+        this._stream = outputStream ?? process.stderr;
+    }
+
+    log(level: string, message?: string, obj?: any) {
+        this._stream.write(
+            JSON.stringify({
+                level: level,
+                message: message,
+                ...obj,
+            } as LogEntry)
+        );
+    }
+}