diff --git a/packages/databricks-sdk-js/src/api-client.ts b/packages/databricks-sdk-js/src/api-client.ts index b73f39c46..c20c7f003 100644 --- a/packages/databricks-sdk-js/src/api-client.ts +++ b/packages/databricks-sdk-js/src/api-client.ts @@ -58,6 +58,11 @@ export class ApiClient { } } + // throw error if the URL is incorrect and we get back an HTML page + if (response.headers.get("content-type")?.match("text/html")) { + throw new HttpError(`Can't connect to ${url.toString()}`, 404); + } + let responseBody = await response.arrayBuffer(); let responseText = new TextDecoder().decode(responseBody); diff --git a/packages/databricks-sdk-js/src/apis/cluster.ts b/packages/databricks-sdk-js/src/apis/cluster.ts index 6ecd13fa8..35958e307 100644 --- a/packages/databricks-sdk-js/src/apis/cluster.ts +++ b/packages/databricks-sdk-js/src/apis/cluster.ts @@ -2,6 +2,33 @@ import {ApiClient} from "../api-client"; +import * as delegate from "./delegate"; + +// +// Enums. +// + +export type AwsAvailability = "SPOT" | "ON_DEMAND" | "SPOT_WITH_FALLBACK"; + +export type AzureAvailability = + | "SPOT_AZURE" + | "ON_DEMAND_AZURE" + | "SPOT_WITH_FALLBACK_AZURE"; + +export type GcpAvailability = + | "PREEMPTIBLE_GCP" + | "ON_DEMAND_GCP" + | "PREEMPTIBLE_WITH_FALLBACK_GCP"; + +export type ClusterSource = + | "UI" + | "JOB" + | "API" + | "SQL" + | "MODELS" + | "PIPELINE" + | "PIPELINE_MAINTENANCE"; + export type ClusterState = | "PENDING" | "RUNNING" @@ -12,152 +39,794 @@ export type ClusterState = | "ERROR" | "UNKNOWN"; -export interface ClustersListRequest {} -export interface ClustersListResponse { - clusters: Array<{ - cluster_id: string; - creator_user_name: string; - spark_context_id: number; - cluster_name: string; - spark_version: string; - node_type_id: string; - driver_node_type_id: string; - state: ClusterState; - start_time: number; - state_message: string; - cluster_memory_mb: number; - cluster_cores: number; - }>; -} - -export interface ClustersCreateRequest { - cluster_name: string; - spark_version: string; - node_type_id: string; - spark_conf?: Record; - aws_attributes?: Record; - custom_tags?: Record; - spark_env_vars?: Record; +export type ListOrder = "DESC" | "ASC"; + +export type TerminationCode = + | "UNKNOWN" + | "USER_REQUEST" + | "JOB_FINISHED" + | "INACTIVITY" + | "CLOUD_PROVIDER_SHUTDOWN" + | "COMMUNICATION_LOST" + | "CLOUD_PROVIDER_LAUNCH_FAILURE" + | "INIT_SCRIPT_FAILURE" + | "SPARK_STARTUP_FAILURE" + | "INVALID_ARGUMENT" + | "UNEXPECTED_LAUNCH_FAILURE" + | "INTERNAL_ERROR" + | "INSTANCE_UNREACHABLE" + | "REQUEST_REJECTED" + | "TRIAL_EXPIRED" + | "DRIVER_UNREACHABLE" + | "SPARK_ERROR" + | "DRIVER_UNRESPONSIVE" + | "METASTORE_COMPONENT_UNHEALTHY" + | "DBFS_COMPONENT_UNHEALTHY" + | "EXECUTION_COMPONENT_UNHEALTHY" + | "AZURE_RESOURCE_MANAGER_THROTTLING" + | "AZURE_RESOURCE_PROVIDER_THROTTLING" + | "NETWORK_CONFIGURATION_FAILURE" + | "CONTAINER_LAUNCH_FAILURE" + | "INSTANCE_POOL_CLUSTER_FAILURE" + | "SKIPPED_SLOW_NODES" + | "ATTACH_PROJECT_FAILURE" + | "UPDATE_INSTANCE_PROFILE_FAILURE" + | "DATABASE_CONNECTION_FAILURE" + | "REQUEST_THROTTLED" + | "SELF_BOOTSTRAP_FAILURE" + | "GLOBAL_INIT_SCRIPT_FAILURE" + | "SLOW_IMAGE_DOWNLOAD" + | "INVALID_SPARK_IMAGE" + | "NPIP_TUNNEL_TOKEN_FAILURE" + | "HIVE_METASTORE_PROVISIONING_FAILURE" + | "AZURE_INVALID_DEPLOYMENT_TEMPLATE" + | "AZURE_UNEXPECTED_DEPLOYMENT_TEMPLATE_FAILURE" + | "SUBNET_EXHAUSTED_FAILURE" + | "BOOTSTRAP_TIMEOUT" + | "STORAGE_DOWNLOAD_FAILURE" + | "CONTROL_PLANE_REQUEST_FAILURE" + | "BOOTSTRAP_TIMEOUT_CLOUD_PROVIDER_EXCEPTION" + | "AWS_INSUFFICIENT_INSTANCE_CAPACITY_FAILURE" + | "DOCKER_IMAGE_PULL_FAILURE" + | "AZURE_VNET_CONFIGURATION_FAILURE" + | "NPIP_TUNNEL_SETUP_FAILURE" + | "AWS_AUTHORIZATION_FAILURE" + | "NEPHOS_RESOURCE_MANAGEMENT" + | "STS_CLIENT_SETUP_FAILURE" + | "SECURITY_DAEMON_REGISTRATION_EXCEPTION" + | "AWS_REQUEST_LIMIT_EXCEEDED" + | "AWS_INSUFFICIENT_FREE_ADDRESSES_IN_SUBNET_FAILURE" + | "AWS_UNSUPPORTED_FAILURE" + | "AZURE_QUOTA_EXCEEDED_EXCEPTION" + | "AZURE_OPERATION_NOT_ALLOWED_EXCEPTION" + | "NFS_MOUNT_FAILURE" + | "K8S_AUTOSCALING_FAILURE" + | "K8S_DBR_CLUSTER_LAUNCH_TIMEOUT" + | "SPARK_IMAGE_DOWNLOAD_FAILURE" + | "AZURE_VM_EXTENSION_FAILURE" + | "WORKSPACE_CANCELLED_ERROR" + | "AWS_MAX_SPOT_INSTANCE_COUNT_EXCEEDED_FAILURE" + | "TEMPORARILY_UNAVAILABLE" + | "WORKER_SETUP_FAILURE" + | "IP_EXHAUSTION_FAILURE" + | "GCP_QUOTA_EXCEEDED" + | "CLOUD_PROVIDER_RESOURCE_STOCKOUT" + | "GCP_SERVICE_ACCOUNT_DELETED" + | "AZURE_BYOK_KEY_PERMISSION_FAILURE" + | "SPOT_INSTANCE_TERMINATION" + | "AZURE_EPHEMERAL_DISK_FAILURE" + | "ABUSE_DETECTED" + | "IMAGE_PULL_PERMISSION_DENIED"; + +export type TerminationParameter = + | "username" + | "aws_api_error_code" + | "aws_instance_state_reason" + | "aws_spot_request_status" + | "aws_spot_request_fault_code" + | "aws_impaired_status_details" + | "aws_instance_status_event" + | "aws_error_message" + | "databricks_error_message" + | "inactivity_duration_min" + | "azure_error_code" + | "azure_error_message" + | "instance_id" + | "instance_pool_id" + | "instance_pool_error_code" + | "invalid_spark_image_prefix" + | "gcp_error_code" + | "gcp_error_message"; + +export type TerminationType = + | "SUCCESS" + | "CLIENT_ERROR" + | "SERVICE_FAULT" + | "CLOUD_FAILURE"; + +export type ClusterEventType = + | "CREATING" + | "STARTING" + | "RESTARTING" + | "TERMINATING" + | "EDITED" + | "RUNNING" + | "RESIZING" + | "NODES_LOST" + | "UPSIZE_COMPLETED" + | "INIT_SCRIPTS_STARTED" + | "INIT_SCRIPTS_FINISHED" + | "DID_NOT_EXPAND_DISK" + | "EXPANDED_DISK" + | "FAILED_TO_EXPAND_DISK" + | "DRIVER_HEALTHY" + | "DRIVER_NOT_RESPONDING" + | "DRIVER_UNAVAILABLE" + | "SPARK_EXCEPTION" + | "METASTORE_DOWN" + | "DBFS_DOWN" + | "AUTOSCALING_STATS_REPORT" + | "NODE_BLACKLISTED" + | "PINNED" + | "UNPINNED" + | "NODE_EXCLUDED_DECOMMISSIONED"; + +export type ResizeCause = + | "AUTOSCALE" + | "USER_REQUEST" + | "AUTORECOVERY" + | "REPLACE_BAD_NODES"; + +export type EbsVolumeType = "GENERAL_PURPOSE_SSD" | "THROUGHPUT_OPTIMIZED_HDD"; + +export type AzureDiskVolumeType = "PREMIUM_LRS" | "STANDARD_LRS"; + +export type InstancePoolState = "ACTIVE" | "STOPPED" | "DELETED"; + +// +// Subtypes used in request/response types. +// + +export interface ClusterInfo { + cluster_id?: string; + creator_user_name?: string; + driver?: SparkNode; + executors?: Array; + spark_context_id?: number; + jdbc_port?: number; + cluster_name?: string; + spark_version?: string; + spark_conf?: Array; + aws_attributes?: AwsAttributes; + azure_attributes?: AzureAttributes; + gcp_attributes?: GcpAttributes; + node_type_id?: string; + driver_node_type_id?: string; + ssh_public_keys?: Array; + custom_tags?: Array; + cluster_log_conf?: ClusterLogConf; + spark_env_vars?: Array; autotermination_minutes?: number; - init_scripts?: Array; //TODO + enable_elastic_disk?: boolean; + cluster_source?: ClusterSource; + instance_pool_id?: string; policy_id?: string; - num_workers: number; - cluster_log_conf?: any; // TODO - autoscale?: { - min_workers: number; - max_workers: number; - }; + enable_local_disk_encryption?: boolean; + driver_instance_pool_id?: string; + state?: ClusterState; + state_message?: string; + start_time?: number; + terminated_time?: number; + last_state_loss_time?: number; + last_restarted_time?: number; + num_workers?: number; + autoscale?: AutoScale; + cluster_memory_mb?: number; + cluster_cores?: number; + default_tags?: Array; + cluster_log_status?: LogSyncStatus; + termination_reason?: TerminationReason; +} + +export interface ClusterAttributes { + cluster_name?: string; + spark_version?: string; + spark_conf?: Array; + aws_attributes?: AwsAttributes; + azure_attributes?: AzureAttributes; + gcp_attributes?: GcpAttributes; + node_type_id?: string; + driver_node_type_id?: string; ssh_public_keys?: Array; - // TODO more fields missing here + custom_tags?: Array; + cluster_log_conf?: ClusterLogConf; + spark_env_vars?: Array; + autotermination_minutes?: number; + enable_elastic_disk?: boolean; + cluster_source?: ClusterSource; + instance_pool_id?: string; + policy_id?: string; + enable_local_disk_encryption?: boolean; + driver_instance_pool_id?: string; +} + +export interface ClusterTag { + key?: string; + value?: string; +} + +export interface AwsAttributes { + first_on_demand?: number; + availability?: AwsAvailability; + zone_id?: string; + instance_profile_arn?: string; + spot_bid_price_percent?: number; + ebs_volume_type?: EbsVolumeType; + ebs_volume_count?: number; + ebs_volume_size?: number; + ebs_volume_iops?: number; + ebs_volume_throughput?: number; +} + +export interface AzureAttributes { + log_analytics_info?: LogAnalyticsInfo; + first_on_demand?: number; + availability?: AzureAvailability; + spot_bid_max_price?: number; +} + +export interface GcpAttributes { + google_service_account?: string; + boot_disk_size?: number; + availability?: GcpAvailability; +} + +export interface LogAnalyticsInfo { + log_analytics_workspace_id?: string; + log_analytics_primary_key?: string; +} + +export interface ClusterSize { + num_workers?: number; + autoscale?: AutoScale; +} + +export interface AutoScale { + min_workers?: number; + max_workers?: number; +} + +export interface SparkInfo {} + +export interface SparkNode { + private_ip?: string; + public_dns?: string; + node_id?: string; + instance_id?: string; + start_timestamp?: number; + node_aws_attributes?: SparkNodeAwsAttributes; + host_private_ip?: string; +} + +export interface SparkNodeAwsAttributes { + is_spot?: boolean; } -export interface ClustersCreateResponse { +export interface SparkVersion { + key?: string; + name?: string; +} + +export interface SparkConfPair { + key?: string; + value?: string; +} + +export interface SparkEnvPair { + key?: string; + value?: string; +} + +export interface DbfsStorageInfo { + destination?: string; +} + +export interface S3StorageInfo { + destination?: string; + region?: string; + endpoint?: string; + enable_encryption?: boolean; + encryption_type?: string; + kms_key?: string; + canned_acl?: string; +} + +export interface ClusterLogConf { + dbfs?: DbfsStorageInfo; + s3?: S3StorageInfo; +} + +export interface LogSyncStatus { + last_attempted?: number; + last_exception?: string; +} + +export interface GetInstance {} + +export interface Policy { + policy_id?: string; + name?: string; + definition?: string; + creator_user_name?: string; + created_at_timestamp?: number; +} + +export interface TerminationReason { + code?: TerminationCode; + type?: TerminationType; + parameters?: Array; +} + +export interface ParameterPair { + key?: TerminationParameter; + value?: string; +} + +export interface InstancePoolAndStats { + instance_pool_name?: string; + min_idle_instances?: number; + max_capacity?: number; + aws_attributes?: InstancePoolAwsAttributes; + node_type_id?: string; + custom_tags?: Array; + idle_instance_autotermination_minutes?: number; + enable_elastic_disk?: boolean; + disk_spec?: DiskSpec; + preloaded_docker_images?: Array; + preloaded_spark_versions?: Array; + azure_attributes?: InstancePoolAzureAttributes; + gcp_attributes?: InstancePoolGcpAttributes; + instance_pool_id?: string; + default_tags?: Array; + state?: InstancePoolState; + stats?: InstancePoolStats; + status?: InstancePoolStatus; +} + +export interface ClusterEvent { cluster_id: string; + timestamp?: number; + type?: ClusterEventType; + details?: EventDetails; +} + +export interface EventDetails { + current_num_workers?: number; + target_num_workers?: number; + previous_attributes?: ClusterAttributes; + attributes?: ClusterAttributes; + previous_cluster_size?: ClusterSize; + cluster_size?: ClusterSize; + cause?: ResizeCause; + reason?: TerminationReason; + user?: string; + previous_disk_size?: number; + disk_size?: number; + free_space?: number; + instance_id?: string; + did_not_expand_reason?: string; + driver_state_message?: string; + job_run_name?: string; + enable_termination_for_node_blocklisted?: boolean; + current_num_vcpus?: number; + target_num_vcpus?: number; +} + +export interface EbsVolume {} + +export interface AzureDiskVolume {} + +export interface NodeType { + node_type_id: string; + memory_mb: number; + num_cores: number; + description: string; + instance_type_id: string; + is_deprecated?: boolean; + is_encrypted_in_transit?: boolean; +} + +export interface DiskSpec { + disk_type?: DiskType; + disk_count?: number; + disk_size?: number; + disk_iops?: number; + disk_throughput?: number; +} + +export interface DiskType { + ebs_volume_type?: EbsVolumeType; + azure_disk_volume_type?: AzureDiskVolumeType; +} + +export interface DockerImage { + url?: string; + basic_auth?: DockerBasicAuth; +} + +export interface DockerBasicAuth { + username?: string; + password?: string; +} + +export interface InstancePool {} + +export interface InstancePoolGcpAttributes { + gcp_availability?: GcpAvailability; +} + +export interface InstancePoolAwsAttributes { + availability?: AwsAvailability; + zone_id?: string; + spot_bid_price_percent?: number; +} + +export interface InstancePoolAzureAttributes { + availability?: AzureAvailability; + spot_bid_max_price?: number; +} + +export interface InstancePoolStats { + used_count?: number; + idle_count?: number; + pending_used_count?: number; + pending_idle_count?: number; +} + +export interface InstancePoolStatus { + pending_instance_errors?: Array; +} + +export interface PendingInstanceError { + instance_id?: string; + message?: string; +} + +// +// Request/response types. +// + +export interface ListClustersRequest {} + +export interface ListClustersResponse { + clusters?: Array; } -export interface ClustersTerminateRequest { +export interface CreateClusterRequest { + num_workers?: number; + autoscale?: AutoScale; + cluster_name?: string; + spark_version?: string; + spark_conf?: Array; + aws_attributes?: AwsAttributes; + azure_attributes?: AzureAttributes; + gcp_attributes?: GcpAttributes; + node_type_id?: string; + driver_node_type_id?: string; + ssh_public_keys?: Array; + custom_tags?: Array; + cluster_log_conf?: ClusterLogConf; + spark_env_vars?: Array; + autotermination_minutes?: number; + enable_elastic_disk?: boolean; + cluster_source?: ClusterSource; + instance_pool_id?: string; + policy_id?: string; + enable_local_disk_encryption?: boolean; + driver_instance_pool_id?: string; + apply_policy_default_values?: boolean; +} + +export interface CreateClusterResponse { + cluster_id?: string; +} + +export interface StartClusterRequest { cluster_id: string; } -export interface ClustersTerminateResponse {} +export interface StartClusterResponse {} -export interface ClustersDeleteRequest { +export interface GetSparkVersionsRequest {} + +export interface GetSparkVersionsResponse { + versions?: Array; +} + +export interface DeleteClusterRequest { cluster_id: string; } -export interface ClustersDeleteResponse {} +export interface DeleteClusterResponse {} -export interface ClustersGetRequest { +export interface PermanentDeleteClusterRequest { cluster_id: string; } -export interface ClustersStartRequest { +export interface PermanentDeleteClusterResponse {} + +export interface RestartClusterRequest { cluster_id: string; } -export interface ClustersStartResponse {} +export interface RestartClusterResponse {} -export interface ClustersGetResponse { +export interface ResizeClusterRequest { cluster_id: string; - spark_context_id: string; - cluster_source: "UI" | "JOB" | "API"; - state: ClusterState; - state_message: string; + num_workers?: number; + autoscale?: AutoScale; +} - cluster_name: string; - spark_version: string; - node_type_id: string; - spark_conf?: Record; - aws_attributes?: Record; - custom_tags?: Record; - spark_env_vars?: Record; +export interface ResizeClusterResponse {} + +export interface EditClusterRequest { + cluster_id: string; + num_workers?: number; + autoscale?: AutoScale; + cluster_name?: string; + spark_version?: string; + spark_conf?: Array; + aws_attributes?: AwsAttributes; + azure_attributes?: AzureAttributes; + gcp_attributes?: GcpAttributes; + node_type_id?: string; + driver_node_type_id?: string; + ssh_public_keys?: Array; + custom_tags?: Array; + cluster_log_conf?: ClusterLogConf; + spark_env_vars?: Array; autotermination_minutes?: number; - init_scripts?: Array; //TODO + enable_elastic_disk?: boolean; + cluster_source?: ClusterSource; + instance_pool_id?: string; policy_id?: string; - num_workers: number; - cluster_log_conf?: any; // TODO - autoscale?: { - min_workers: number; - max_workers: number; - }; + enable_local_disk_encryption?: boolean; + driver_instance_pool_id?: string; + apply_policy_default_values?: boolean; +} + +export interface EditClusterResponse {} + +export interface GetClusterRequest { + cluster_id: string; +} + +export interface GetClusterResponse { + cluster_id?: string; + creator_user_name?: string; + driver?: SparkNode; + executors?: Array; + spark_context_id?: number; + jdbc_port?: number; + cluster_name?: string; + spark_version?: string; + spark_conf?: Array; + aws_attributes?: AwsAttributes; + azure_attributes?: AzureAttributes; + gcp_attributes?: GcpAttributes; + node_type_id?: string; + driver_node_type_id?: string; ssh_public_keys?: Array; - // TODO more fields missing here + custom_tags?: Array; + cluster_log_conf?: ClusterLogConf; + spark_env_vars?: Array; + autotermination_minutes?: number; + enable_elastic_disk?: boolean; + cluster_source?: ClusterSource; + instance_pool_id?: string; + policy_id?: string; + enable_local_disk_encryption?: boolean; + driver_instance_pool_id?: string; + state?: ClusterState; + state_message?: string; + start_time?: number; + terminated_time?: number; + last_state_loss_time?: number; + last_restarted_time?: number; + num_workers?: number; + autoscale?: AutoScale; + cluster_memory_mb?: number; + cluster_cores?: number; + default_tags?: Array; + cluster_log_status?: LogSyncStatus; + termination_reason?: TerminationReason; +} + +export interface PinClusterRequest { + cluster_id: string; +} + +export interface PinClusterResponse {} + +export interface UnpinClusterRequest { + cluster_id: string; +} + +export interface UnpinClusterResponse {} + +export interface ListNodeTypesRequest {} + +export interface ListNodeTypesResponse { + node_types?: Array; + success?: delegate.CpalSuccessResponse; + failure?: delegate.CpalFailureResponse; +} + +export interface ListAvailableZonesRequest {} + +export interface ListAvailableZonesResponse { + zones?: Array; + default_zone?: string; +} + +export interface GetEventsRequest { + cluster_id: string; + start_time?: number; + end_time?: number; + order?: ListOrder; + event_types?: Array; + offset?: number; + limit?: number; +} + +export interface GetEventsResponse { + events?: Array; + next_page?: GetEventsRequest; + total_count?: number; } -export class ClustersApi { +export class ClusterService { readonly client: ApiClient; constructor(client: ApiClient) { this.client = client; } - async list(req: ClustersListRequest): Promise { + async listClusters( + request: ListClustersRequest + ): Promise { return (await this.client.request( "/api/2.0/clusters/list", "GET", - req - )) as ClustersListResponse; + request + )) as ListClustersResponse; } - async create(req: ClustersCreateRequest): Promise { + async create( + request: CreateClusterRequest + ): Promise { return (await this.client.request( - "api/2.0/clusters/create", + "/api/2.0/clusters/create", "POST", - req - )) as ClustersCreateResponse; + request + )) as CreateClusterResponse; } - async terminate( - req: ClustersTerminateRequest - ): Promise { + async start(request: StartClusterRequest): Promise { return (await this.client.request( - "api/2.0/clusters/delete", + "/api/2.0/clusters/start", "POST", - req - )) as ClustersTerminateResponse; + request + )) as StartClusterResponse; } - async delete(req: ClustersDeleteRequest): Promise { + async listSparkVersions( + request: GetSparkVersionsRequest + ): Promise { return (await this.client.request( - "api/2.0/clusters/permanent-delete", + "/api/2.0/clusters/spark-versions", + "GET", + request + )) as GetSparkVersionsResponse; + } + + async delete( + request: DeleteClusterRequest + ): Promise { + return (await this.client.request( + "/api/2.0/clusters/delete", + "POST", + request + )) as DeleteClusterResponse; + } + + async permanentDelete( + request: PermanentDeleteClusterRequest + ): Promise { + return (await this.client.request( + "/api/2.0/clusters/permanent-delete", "POST", - req - )) as ClustersDeleteResponse; + request + )) as PermanentDeleteClusterResponse; + } + + async restart( + request: RestartClusterRequest + ): Promise { + return (await this.client.request( + "/api/2.0/clusters/restart", + "POST", + request + )) as RestartClusterResponse; + } + + async resize( + request: ResizeClusterRequest + ): Promise { + return (await this.client.request( + "/api/2.0/clusters/resize", + "POST", + request + )) as ResizeClusterResponse; + } + + async edit(request: EditClusterRequest): Promise { + return (await this.client.request( + "/api/2.0/clusters/edit", + "POST", + request + )) as EditClusterResponse; + } + + async get(request: GetClusterRequest): Promise { + return (await this.client.request( + "/api/2.0/clusters/get", + "GET", + request + )) as GetClusterResponse; + } + + async pin(request: PinClusterRequest): Promise { + return (await this.client.request( + "/api/2.0/clusters/pin", + "POST", + request + )) as PinClusterResponse; + } + + async unpin(request: UnpinClusterRequest): Promise { + return (await this.client.request( + "/api/2.0/clusters/unpin", + "POST", + request + )) as UnpinClusterResponse; + } + + async listNodeTypes( + request: ListNodeTypesRequest + ): Promise { + return (await this.client.request( + "/api/2.0/clusters/list-node-types", + "GET", + request + )) as ListNodeTypesResponse; } - async get(req: ClustersGetRequest): Promise { + async listAvailableZones( + request: ListAvailableZonesRequest + ): Promise { return (await this.client.request( - "api/2.0/clusters/get", + "/api/2.0/clusters/list-zones", "GET", - req - )) as ClustersGetResponse; + request + )) as ListAvailableZonesResponse; } - async start(req: ClustersStartRequest): Promise { + async getEvents(request: GetEventsRequest): Promise { return (await this.client.request( - "api/2.0/clusters/start", + "/api/2.0/clusters/events", "POST", - req - )) as ClustersStartResponse; + request + )) as GetEventsResponse; } } diff --git a/packages/databricks-sdk-js/src/apis/delegate.ts b/packages/databricks-sdk-js/src/apis/delegate.ts new file mode 100644 index 000000000..0411363ca --- /dev/null +++ b/packages/databricks-sdk-js/src/apis/delegate.ts @@ -0,0 +1,5 @@ +/* eslint-disable @typescript-eslint/naming-convention */ + +export interface CpalSuccessResponse {} + +export interface CpalFailureResponse {} diff --git a/packages/databricks-sdk-js/src/apis/jobs.integ.ts b/packages/databricks-sdk-js/src/apis/jobs.integ.ts index 7d1844d4a..ccb5fd092 100644 --- a/packages/databricks-sdk-js/src/apis/jobs.integ.ts +++ b/packages/databricks-sdk-js/src/apis/jobs.integ.ts @@ -56,7 +56,7 @@ describe(__filename, function () { }); // console.log(output); - assert.equal(output.logs, "hello from job"); + assert.equal(output.logs.trim(), "hello from job"); break; } } diff --git a/packages/databricks-sdk-js/src/services/Cluster.integ.ts b/packages/databricks-sdk-js/src/services/Cluster.integ.ts index bcc51ec49..d2c098fc4 100644 --- a/packages/databricks-sdk-js/src/services/Cluster.integ.ts +++ b/packages/databricks-sdk-js/src/services/Cluster.integ.ts @@ -4,7 +4,6 @@ import {Cluster} from ".."; import assert = require("assert"); import {IntegrationTestSetup} from "../test/IntegrationTestSetup"; -import cluster from "cluster"; describe(__filename, function () { let integSetup: IntegrationTestSetup; @@ -40,7 +39,7 @@ describe(__filename, function () { let clusterB = await Cluster.fromClusterName( integSetup.client, - clusterA.details.cluster_name + clusterA.details.cluster_name! ); assert(clusterA.details.cluster_id); diff --git a/packages/databricks-sdk-js/src/services/Cluster.ts b/packages/databricks-sdk-js/src/services/Cluster.ts index 4d55f7e9d..2ef498b4d 100644 --- a/packages/databricks-sdk-js/src/services/Cluster.ts +++ b/packages/databricks-sdk-js/src/services/Cluster.ts @@ -1,21 +1,53 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {ApiClient} from "../api-client"; -import {ClustersApi, ClustersGetResponse, ClusterState} from "../apis/cluster"; +import { + ClusterService, + GetClusterResponse, + ClusterState, +} from "../apis/cluster"; import {ExecutionContext} from "./ExecutionContext"; export class Cluster { - private clusterApi: ClustersApi; + private clusterApi: ClusterService; constructor( private client: ApiClient, - private clusterDetails: ClustersGetResponse + private clusterDetails: GetClusterResponse ) { - this.clusterApi = new ClustersApi(client); + this.clusterApi = new ClusterService(client); + } + + get id(): string { + return this.clusterDetails.cluster_id!; + } + + get name(): string { + return this.clusterDetails.cluster_name!; + } + + get memoryMb(): number | undefined { + return this.clusterDetails.cluster_memory_mb; + } + + get cores(): number | undefined { + return this.clusterDetails.cluster_cores; + } + + get sparkVersion(): string { + return this.clusterDetails.spark_version!; + } + + get creator(): string { + return this.clusterDetails.creator_user_name || ""; } get state(): ClusterState { - return this.clusterDetails.state; + return this.clusterDetails.state!; + } + + get stateMessage(): string { + return this.clusterDetails.state_message || ""; } get details() { @@ -24,19 +56,19 @@ export class Cluster { async refresh() { this.clusterDetails = await this.clusterApi.get({ - cluster_id: this.clusterDetails.cluster_id, + cluster_id: this.clusterDetails.cluster_id!, }); } async start() { await this.clusterApi.start({ - cluster_id: this.clusterDetails.cluster_id, + cluster_id: this.clusterDetails.cluster_id!, }); } async stop() { - await this.clusterApi.terminate({ - cluster_id: this.clusterDetails.cluster_id, + await this.clusterApi.delete({ + cluster_id: this.clusterDetails.cluster_id!, }); } @@ -55,7 +87,7 @@ export class Cluster { async createExecutioncontext(): Promise { return await ExecutionContext.create( this.client, - this.clusterDetails.cluster_id + this.clusterDetails.cluster_id! ); } @@ -79,16 +111,16 @@ export class Cluster { client: ApiClient, clusterName: string ): Promise { - let clusterApi = new ClustersApi(client); - let clusterList = await clusterApi.list({}); - let cluster = clusterList.clusters.find((cluster) => { + let clusterApi = new ClusterService(client); + let clusterList = await clusterApi.listClusters({}); + let cluster = clusterList.clusters?.find((cluster) => { return cluster.cluster_name === clusterName; }); if (!cluster) { return; } - let response = await clusterApi.get({cluster_id: cluster.cluster_id}); + let response = await clusterApi.get({cluster_id: cluster.cluster_id!}); return new Cluster(client, response); } @@ -96,8 +128,19 @@ export class Cluster { client: ApiClient, clusterId: string ): Promise { - let clusterApi = new ClustersApi(client); + let clusterApi = new ClusterService(client); let response = await clusterApi.get({cluster_id: clusterId}); return new Cluster(client, response); } + + static async list(client: ApiClient): Promise> { + let clusterApi = new ClusterService(client); + let response = await clusterApi.listClusters({}); + + if (!response.clusters) { + return []; + } + + return response.clusters.map((c) => new Cluster(client, c)); + } } diff --git a/packages/databricks-sdk-js/src/test/IntegrationTestSetup.ts b/packages/databricks-sdk-js/src/test/IntegrationTestSetup.ts index 4a64f7554..762d3d1e8 100644 --- a/packages/databricks-sdk-js/src/test/IntegrationTestSetup.ts +++ b/packages/databricks-sdk-js/src/test/IntegrationTestSetup.ts @@ -2,7 +2,7 @@ import {v4 as uuidv4} from "uuid"; import {ApiClient} from "../api-client"; -import {ClustersApi} from "../apis/cluster"; +import {ClusterService} from "../apis/cluster"; import {fromEnv} from "../auth/fromEnv"; export class IntegrationTestSetup { @@ -16,8 +16,8 @@ export class IntegrationTestSetup { static async getInstance(): Promise { if (!this._instance) { let clusterId; - let client = new ApiClient(fromEnv()); - let clustersApi = new ClustersApi(client); + let client = new ApiClient(); + let clustersApi = new ClusterService(client); if (!process.env["DATABRICKS_CLUSTER_ID"]) { throw new Error( diff --git a/packages/databricks-vscode/src/cluster/ClusterListDataProvider.ts b/packages/databricks-vscode/src/cluster/ClusterListDataProvider.ts index 6b2f80e43..ecf17a1d7 100644 --- a/packages/databricks-vscode/src/cluster/ClusterListDataProvider.ts +++ b/packages/databricks-vscode/src/cluster/ClusterListDataProvider.ts @@ -8,20 +8,20 @@ import { TreeItem, TreeItemCollapsibleState, } from "vscode"; -import {ClusterModel, ClusterNode} from "./ClusterModel"; +import {Cluster} from "@databricks/databricks-sdk"; +import {ClusterModel} from "./ClusterModel"; /** * Data provider for the cluster tree view */ export class ClusterListDataProvider - implements TreeDataProvider, Disposable + implements TreeDataProvider, Disposable { private _onDidChangeTreeData: EventEmitter< - ClusterNode | TreeItem | undefined | void - > = new EventEmitter(); - readonly onDidChangeTreeData: Event< - ClusterNode | TreeItem | undefined | void - > = this._onDidChangeTreeData.event; + Cluster | TreeItem | undefined | void + > = new EventEmitter(); + readonly onDidChangeTreeData: Event = + this._onDidChangeTreeData.event; private disposables: Array; @@ -38,11 +38,15 @@ export class ClusterListDataProvider this.disposables.forEach((d) => d.dispose()); } - getTreeItem(element: ClusterNode | TreeItem): TreeItem { + getTreeItem(element: Cluster | TreeItem): TreeItem { if (!this.isClusterNode(element)) { return element; } + if (element instanceof TreeItem) { + return element; + } + let icon: ThemeIcon; switch (element.state) { case "RUNNING": @@ -76,10 +80,8 @@ export class ClusterListDataProvider }; } - private isClusterNode( - element: ClusterNode | TreeItem - ): element is ClusterNode { - return (element as ClusterNode).state !== undefined; + private isClusterNode(element: Cluster | TreeItem): element is Cluster { + return (element as Cluster).state !== undefined; } private autoReload(refreshRateInMs: number): Disposable { @@ -94,8 +96,8 @@ export class ClusterListDataProvider } getChildren( - element?: ClusterNode | TreeItem | undefined - ): ProviderResult> { + element?: Cluster | TreeItem | undefined + ): ProviderResult> { if (element) { if (this.isClusterNode(element)) { let children = [ diff --git a/packages/databricks-vscode/src/cluster/ClusterModel.ts b/packages/databricks-vscode/src/cluster/ClusterModel.ts index a7a4207ae..9b438d4c3 100644 --- a/packages/databricks-vscode/src/cluster/ClusterModel.ts +++ b/packages/databricks-vscode/src/cluster/ClusterModel.ts @@ -1,18 +1,7 @@ -import {ClustersApi, ClusterState} from "@databricks/databricks-sdk"; -import {Disposable, Event, EventEmitter, TreeItem} from "vscode"; +import {Cluster} from "@databricks/databricks-sdk"; +import {Disposable, Event, EventEmitter} from "vscode"; import {ConnectionManager} from "../configuration/ConnectionManager"; -export interface ClusterNode { - id: string; - name: string; - memoryMb: number; - cores: number; - sparkVersion: string; - creator: string; - state: ClusterState; - stateMessage: string; -} - export type ClusterFilter = "ALL" | "ME" | "RUNNING"; /** @@ -27,7 +16,7 @@ export class ClusterModel implements Disposable { private disposables: Array; private _filter: ClusterFilter = "ALL"; - private _clusters: Promise | undefined> = + private _clusters: Promise | undefined> = Promise.resolve(undefined); private _dirty = true; @@ -51,7 +40,7 @@ export class ClusterModel implements Disposable { this._onDidChange.fire(); } - public get roots(): Promise { + public get roots(): Promise { if (this._dirty) { this._clusters = this.loadClusters(); this._dirty = false; @@ -60,33 +49,19 @@ export class ClusterModel implements Disposable { return this.applyFilter(this._clusters); } - private async loadClusters(): Promise | undefined> { + private async loadClusters(): Promise | undefined> { let apiClient = this.connectionManager.apiClient; if (!apiClient) { return; } - let clusterService = new ClustersApi(apiClient); - let clusters = await clusterService.list({}); - - return clusters.clusters.map((c) => { - return { - name: c.cluster_name, - id: c.cluster_id, - memoryMb: c.cluster_memory_mb, - cores: c.cluster_cores, - sparkVersion: c.spark_version, - creator: c.creator_user_name, - state: c.state, - stateMessage: c.state_message, - }; - }); + return await Cluster.list(apiClient); } private async applyFilter( - nodesPromise: Promise | undefined> - ): Promise | undefined> { + nodesPromise: Promise | undefined> + ): Promise | undefined> { const nodes = await nodesPromise; if (!nodes) {