Skip to content

Commit 4d1b0f6

Browse files
kartikgupta-dbkartikgupta-db
authored
[DECO-228] Format output logs and also output to a file (#153)
Also setup basic vscode settings options for log length etc ## Truncation Depth = 2 ```js { logger: 'SDK', operationId: 'cf9f7fd8-bb44-446b-a694-76c9477b44db', operationName: 'ClustersService.list', loggingFunction: 'ApiClient.request', timestamp: '11/2/2022, 11:51:42 PM', request: { method: 'GET' }, response: { clusters: '[{"cluster_id":"1017-064207-v53fc7dm","c ...(31937 bytes more)' }, level: 'debug', message: 'https://adb-309687753508875.15.azuredatabricks.net/api/2.0/clusters/list?can_use_client=' } ``` ## Truncation Depth = 3 ```js { logger: 'SDK', operationId: '2e97c02c-f8da-4bcb-8cfc-bfba71b1fe47', operationName: 'ClustersService.list', loggingFunction: 'ApiClient.request', timestamp: '11/2/2022, 11:54:11 PM', request: { method: 'GET' }, response: { clusters: [ '{"cluster_id":"1017-064207-v53fc7dm","cr ...(1310 bytes more)', '{"cluster_id":"0818-155213-scc3x4dj","cr ...(1329 bytes more)', '...20 more items' ] }, level: 'debug', message: 'https://adb-309687753508875.15.azuredatabricks.net/api/2.0/clusters/list?can_use_client=' } ``` ## Truncation Depth = 4 ```js { logger: "SDK", operationId: "5a670a2c-dd12-4b95-8390-76bd727452e1", operationName: "ClustersService.list", loggingFunction: "ApiClient.request", timestamp: "11/2/2022, 11:54:42 PM", request: { method: "GET" }, response: { clusters: [ { cluster_id: "1017-064207-v53fc7dm", creator_user_name: "61b77d30-bc10-4214-9650-29cf5db0e941", spark_context_id: "4923128785231340000", cluster_name: "sdk-go-cluster-heabciilbkij", spark_version: "10.4.x-scala2.12", spark_conf: '{"spark.databricks.delta.preview.enabled ...(9 bytes more)', azure_attributes: '{"first_on_demand":1,"availability":"ON_ ...(38 bytes more)', node_type_id: "Standard_F4", driver_node_type_id: "Standard_F4", autotermination_minutes: "10", enable_elastic_disk: "true", disk_spec: "{}", cluster_source: "UI", enable_local_disk_encryption: "false", instance_source: '{"node_type_id":"Standard_F4"}', driver_instance_source: '{"node_type_id":"Standard_F4"}', effective_spark_version: "10.4.x-scala2.12", state: "TERMINATED", state_message: "Inactive cluster terminated (inactive fo ...(14 bytes more)", start_time: "1665988927301", terminated_time: "1665989855187", last_state_loss_time: "1665989248506", last_activity_time: "1665989229258", last_restarted_time: "1665989248621", num_workers: "2", default_tags: '{"Vendor":"Databricks","Creator":"61b77d ...(159 bytes more)', termination_reason: '{"code":"INACTIVITY","type":"SUCCESS","p ...(44 bytes more)', init_scripts_safe_mode: "false", }, { cluster_id: "0818-155213-scc3x4dj", creator_user_name: "serge.smertin@databricks.com", spark_context_id: "6399405132010977000", cluster_name: "Default Test Cluster", spark_version: "11.3.x-scala2.12", spark_conf: '{"spark.databricks.delta.preview.enabled ...(9 bytes more)', azure_attributes: '{"first_on_demand":1,"availability":"ON_ ...(38 bytes more)', node_type_id: "Standard_L8as_v3", driver_node_type_id: "Standard_DS3_v2", autotermination_minutes: "60", enable_elastic_disk: "true", disk_spec: "{}", cluster_source: "UI", enable_local_disk_encryption: "false", instance_source: '{"node_type_id":"Standard_L8as_v3"}', driver_instance_source: '{"node_type_id":"Standard_DS3_v2"}', effective_spark_version: "11.3.x-scala2.12", state: "TERMINATED", state_message: "Inactive cluster terminated (inactive fo ...(14 bytes more)", start_time: "1660837933746", terminated_time: "1667412178375", last_state_loss_time: "1667408387771", last_activity_time: "1667408547018", last_restarted_time: "1667408387822", autoscale: '{"min_workers":1,"max_workers":50}', default_tags: '{"Vendor":"Databricks","Creator":"serge. ...(144 bytes more)', termination_reason: '{"code":"INACTIVITY","type":"SUCCESS","p ...(44 bytes more)', init_scripts_safe_mode: "false", }, "...20 more items", ], }, level: "debug", message: "https://adb-309687753508875.15.azuredatabricks.net/api/2.0/clusters/list?can_use_client=", } ``` <img width="820" alt="Screenshot 2022-11-03 at 16 51 02" src="https://user-images.githubusercontent.com/88345179/199708342-c876310e-1493-485a-946e-78c238682cc6.png">
1 parent 58690e0 commit 4d1b0f6

10 files changed

Lines changed: 198 additions & 48 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,4 +37,4 @@
3737
"ts-mockito": "^2.6.1",
3838
"typescript": "^4.8.4"
3939
}
40-
}
40+
}

‎packages/databricks-sdk-js/src/Redactor.ts‎

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,31 +15,19 @@ export class Redactor {
1515
this.fieldNames.push(fieldName);
1616
}
1717

18-
sanitize(
19-
obj?: any,
20-
dropFields: string[] = [],
21-
maxFieldLength: number = 96
22-
): any {
18+
sanitize(obj?: any, dropFields: string[] = []): any {
2319
if (obj === undefined) {
2420
return undefined;
2521
}
2622

2723
if (isPrimitveType(obj)) {
28-
if (typeof obj === "string") {
29-
return onlyNBytes(obj, maxFieldLength);
30-
}
31-
if (obj instanceof String) {
32-
return onlyNBytes(obj.toString(), maxFieldLength);
33-
}
3424
return obj;
3525
}
36-
3726
if (Array.isArray(obj)) {
38-
return obj.map((e) => this.sanitize(e, dropFields, maxFieldLength));
27+
return obj.map((e) => this.sanitize(e, dropFields));
3928
}
40-
4129
//make a copy of the object
42-
obj = JSON.parse(JSON.stringify(obj));
30+
obj = Object.assign({}, obj);
4331
for (let key in obj) {
4432
if (dropFields.includes(key)) {
4533
delete obj[key];
@@ -49,7 +37,7 @@ export class Redactor {
4937
) {
5038
obj[key] = "***REDACTED***";
5139
} else {
52-
obj[key] = this.sanitize(obj[key], dropFields, maxFieldLength);
40+
obj[key] = this.sanitize(obj[key], dropFields);
5341
}
5442
}
5543

‎packages/databricks-sdk-js/src/logging/NamedLogger.ts‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ const loggers = new Map<string, LoggerDetails>();
1515

1616
export interface LoggerOpts {
1717
fieldNameDenyList: string[];
18-
maxFieldLength: number;
1918
factory: (name: string) => Logger;
2019
}
2120

@@ -32,17 +31,6 @@ export const defaultOpts: LoggerOpts = {
3231
}
3332
return denyList;
3433
},
35-
get maxFieldLength(): number {
36-
const defaultLength = 96;
37-
if (process.env["DATABRICKS_DEBUG_TRUNCATE_BYTES"]) {
38-
try {
39-
return parseInt(process.env["DATABRICKS_DEBUG_TRUNCATE_BYTES"]);
40-
} catch (e) {
41-
return defaultLength;
42-
}
43-
}
44-
return defaultLength;
45-
},
4634
factory: (name) => new DefaultLogger(),
4735
};
4836

@@ -97,8 +85,7 @@ export class NamedLogger {
9785
log(level: string, message?: string, meta?: any) {
9886
meta = defaultRedactor.sanitize(
9987
meta,
100-
this._loggerOpts?.fieldNameDenyList,
101-
this._loggerOpts?.maxFieldLength
88+
this._loggerOpts?.fieldNameDenyList
10289
);
10390

10491
this._logger?.log(level, message, {

‎packages/databricks-vscode-types/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,4 @@
2424
"dependencies": {
2525
"@databricks/databricks-sdk": "workspace:^"
2626
}
27-
}
27+
}

‎packages/databricks-vscode/package.json‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,37 @@
487487
}
488488
]
489489
}
490+
],
491+
"configuration": [
492+
{
493+
"title": "Databricks",
494+
"properties": {
495+
"databricks.logs.maxFieldLength": {
496+
"title": "Max Field Length",
497+
"type": "number",
498+
"default": 40,
499+
"description": "The maximum length of each field displayed in logs outputs panel."
500+
},
501+
"databricks.logs.truncationDepth": {
502+
"title": "Truncation Depth",
503+
"type": "number",
504+
"default": 2,
505+
"description": "The max depth of logs to show without truncation."
506+
},
507+
"databricks.logs.maxArrayLength": {
508+
"title": "Max Array Length",
509+
"type": "number",
510+
"default": 2,
511+
"description": "The maximum number of items to show for array fields."
512+
},
513+
"databricks.logs.enabled": {
514+
"title": "Enabled",
515+
"type": "boolean",
516+
"default": true,
517+
"description": "Enable/disable logging. Reload window for changes to take effect."
518+
}
519+
}
520+
}
490521
]
491522
},
492523
"vsce": {
@@ -517,8 +548,10 @@
517548
"dependencies": {
518549
"@databricks/databricks-sdk": "*",
519550
"@databricks/databricks-vscode-types": "workspace:^",
551+
"@types/triple-beam": "^1.3.2",
520552
"@vscode/debugadapter": "^1.58.0",
521-
"@vscode/webview-ui-toolkit": "^1.1.0"
553+
"@vscode/webview-ui-toolkit": "^1.1.0",
554+
"triple-beam": "^1.3.0"
522555
},
523556
"devDependencies": {
524557
"@istanbuljs/nyc-config-typescript": "^1.0.2",
@@ -565,4 +598,4 @@
565598
],
566599
"report-dir": "coverage"
567600
}
568-
}
601+
}

‎packages/databricks-vscode/src/extension.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ export function activate(context: ExtensionContext): PublicApi | undefined {
4343
*/
4444
return undefined;
4545
}
46-
initLoggers();
46+
initLoggers(workspace.workspaceFolders[0].uri.path);
4747

4848
let cli = new CliWrapper(context);
4949
// Configuration group

‎packages/databricks-vscode/src/logger/WorkspaceConfigs.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import {workspace} from "vscode";
2+
3+
export const workspaceConfigs = {
4+
get maxFieldLength() {
5+
return (
6+
workspace
7+
.getConfiguration("databricks")
8+
?.get<number>("logs.maxFieldLength") ?? 40
9+
);
10+
},
11+
get truncationDepth() {
12+
return (
13+
workspace
14+
.getConfiguration("databricks")
15+
?.get<number>("logs.truncationDepth") ?? 2
16+
);
17+
},
18+
get maxArrayLength() {
19+
return (
20+
workspace
21+
.getConfiguration("databricks")
22+
?.get<number>("logs.maxArrayLength") ?? 2
23+
);
24+
},
25+
get loggingEnabled() {
26+
return (
27+
workspace
28+
.getConfiguration("databricks")
29+
?.get<boolean>("logs.enabled") ?? true
30+
);
31+
},
32+
};

‎packages/databricks-vscode/src/logger/loggers.ts‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,31 @@ import {
22
NamedLogger,
33
ExposedLoggers,
44
} from "@databricks/databricks-sdk/dist/logging";
5-
import {OutputChannel, window} from "vscode";
5+
import {window} from "vscode";
66
import {loggers, format, transports} from "winston";
7-
import {OutputConsoleStream} from "./OutputConsoleStream";
7+
import {getOutputConsoleTransport} from "./outputConsoleTransport";
8+
import {unlink, access} from "fs/promises";
9+
import {workspaceConfigs} from "./WorkspaceConfigs";
810

9-
function getOutputConsoleTransport(outputChannel: OutputChannel) {
10-
return new transports.Stream({
11-
stream: new OutputConsoleStream(outputChannel, {
12-
defaultEncoding: "utf-8",
13-
}),
11+
function getFileTransport(filename: string) {
12+
return new transports.File({
13+
format: format.combine(format.timestamp(), format.json()),
14+
filename: filename,
1415
});
1516
}
16-
export function initLoggers() {
17-
const outputChannel = window.createOutputChannel("Databricks Logs", "json");
17+
18+
export async function initLoggers(rootPath: string) {
19+
if (!workspaceConfigs.loggingEnabled) {
20+
return;
21+
}
22+
23+
const outputChannel = window.createOutputChannel("Databricks Logs");
24+
const logFile = `${rootPath}/.databricks/logs.json`;
25+
try {
26+
await access(logFile);
27+
await unlink(logFile);
28+
} catch (e) {}
29+
1830
outputChannel.clear();
1931

2032
NamedLogger.getOrCreate(
@@ -23,8 +35,10 @@ export function initLoggers() {
2335
factory: (name) => {
2436
return loggers.add(name, {
2537
level: "debug",
26-
format: format.json(),
27-
transports: [getOutputConsoleTransport(outputChannel)],
38+
transports: [
39+
getOutputConsoleTransport(outputChannel),
40+
getFileTransport(logFile),
41+
],
2842
});
2943
},
3044
},
@@ -42,8 +56,10 @@ export function initLoggers() {
4256
factory: (name) => {
4357
return loggers.add(name, {
4458
level: "error",
45-
format: format.json(),
46-
transports: [getOutputConsoleTransport(outputChannel)],
59+
transports: [
60+
getOutputConsoleTransport(outputChannel),
61+
getFileTransport(logFile),
62+
],
4763
});
4864
},
4965
},

‎packages/databricks-vscode/src/logger/outputConsoleTransport.ts‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import {OutputChannel, workspace} from "vscode";
2+
import {transports, format} from "winston";
3+
import {OutputConsoleStream} from "./OutputConsoleStream";
4+
import {LEVEL, MESSAGE, SPLAT} from "triple-beam";
5+
import {workspaceConfigs} from "./WorkspaceConfigs";
6+
7+
function processPrimitiveOrString(obj: any) {
8+
let valueStr: string;
9+
if (Object(obj) !== obj) {
10+
valueStr = typeof obj === "string" ? obj : String(obj).toString();
11+
} else {
12+
valueStr = JSON.stringify(obj);
13+
}
14+
15+
return valueStr.length > workspaceConfigs.maxFieldLength
16+
? `${valueStr.slice(0, workspaceConfigs.maxFieldLength)} ...(${
17+
valueStr.length - workspaceConfigs.maxFieldLength
18+
} bytes more)`
19+
: valueStr;
20+
}
21+
22+
function processArray(obj: Array<any>, depth: number): Array<any> {
23+
const finalArr = [];
24+
for (let child of obj) {
25+
finalArr.push(recursiveTruncate(child, depth - 1));
26+
if (finalArr.length === workspaceConfigs.maxArrayLength) {
27+
break;
28+
}
29+
}
30+
if (obj.length > workspaceConfigs.maxArrayLength) {
31+
finalArr.push(
32+
`...${obj.length - workspaceConfigs.maxArrayLength} more items`
33+
);
34+
}
35+
return finalArr;
36+
}
37+
38+
function recursiveTruncate(obj: any, depth: number) {
39+
//If object is of primitive type
40+
if (Object(obj) !== obj || depth === 0) {
41+
return processPrimitiveOrString(obj);
42+
}
43+
44+
if (Array.isArray(obj)) {
45+
return processArray(obj, depth);
46+
}
47+
48+
obj = Object.assign({}, obj);
49+
50+
for (let key in obj) {
51+
obj[key] = recursiveTruncate(obj[key], depth - 1);
52+
}
53+
return obj;
54+
}
55+
56+
export function getOutputConsoleTransport(outputChannel: OutputChannel) {
57+
return new transports.Stream({
58+
format: format.combine(
59+
format((info) => {
60+
const stripped = Object.assign({}, info) as any;
61+
if (stripped[LEVEL] === "error") {
62+
return info;
63+
}
64+
delete stripped[LEVEL];
65+
delete stripped[MESSAGE];
66+
delete stripped[SPLAT];
67+
delete stripped["level"];
68+
delete stripped["message"];
69+
70+
return {
71+
...info,
72+
...recursiveTruncate(
73+
stripped,
74+
workspaceConfigs.truncationDepth
75+
),
76+
timestamp: new Date().toLocaleString(),
77+
};
78+
})(),
79+
format.prettyPrint({depth: workspaceConfigs.truncationDepth})
80+
),
81+
stream: new OutputConsoleStream(outputChannel, {
82+
defaultEncoding: "utf-8",
83+
}),
84+
});
85+
}

‎yarn.lock‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -778,6 +778,13 @@ __metadata:
778778
languageName: node
779779
linkType: hard
780780

781+
"@types/triple-beam@npm:^1.3.2":
782+
version: 1.3.2
783+
resolution: "@types/triple-beam@npm:1.3.2"
784+
checksum: dd7b4a563fb710abc992e5d59eac481bed9e303fada2e276e37b00be31c392e03300ee468e57761e616512872e77935f92472877d0704a19688d15a726cee17b
785+
languageName: node
786+
linkType: hard
787+
781788
"@types/uuid@npm:^8.3.4":
782789
version: 8.3.4
783790
resolution: "@types/uuid@npm:8.3.4"
@@ -1854,6 +1861,7 @@ __metadata:
18541861
"@types/mocha": ^10.0.0
18551862
"@types/node": ^18.11.9
18561863
"@types/tmp": ^0.2.3
1864+
"@types/triple-beam": ^1.3.2
18571865
"@types/vscode": ^1.69.1
18581866
"@typescript-eslint/eslint-plugin": ^5.42.0
18591867
"@typescript-eslint/parser": ^5.42.0
@@ -1869,6 +1877,7 @@ __metadata:
18691877
nyc: ^15.1.0
18701878
prettier: ^2.7.1
18711879
tmp-promise: ^3.0.3
1880+
triple-beam: ^1.3.0
18721881
ts-mockito: ^2.6.1
18731882
ts-node: ^10.9.1
18741883
typescript: ^4.8.4

0 commit comments

Comments
 (0)