Skip to content

Commit 171b09f

Browse files
author
Conner Aldrich
committed
feat(supervisor): parameterize worker-pod and -container securityContext
The two-phase deploy commit (8f98ebb) added hardcoded securityContext blocks at both pod level (`runAsNonRoot: true, runAsUser: 1000, fsGroup: 1000`) and container level (`runAsNonRoot, runAsUser:1000, allowPrivilegeEscalation: false, capabilities.drop: [ALL]`) on every worker pod the supervisor schedules. Hardcoding `runAsUser: 1000` makes the chart unschedulable on OpenShift and other clusters that enforce arbitrary-UID SCC ranges. It's also opinionated: some compliance regimes also require seccompProfile, others forbid it. Replace both with env-driven JSON objects: - KUBERNETES_WORKER_POD_SECURITY_CONTEXT (V1PodSecurityContext) - KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT (V1SecurityContext) Default is empty `{}` for both, which preserves upstream's pre-fork behavior of not setting a securityContext at all. Operators who want the previous fork defaults set them explicitly via their orchestration (e.g. supervisor.extraEnvVars in the Helm chart). Also factor a JsonObjectEnv helper into envUtil.ts so this and the existing KUBERNETES_WORKER_POD_ANNOTATIONS env var share validation logic. Adds JsonStringMap and JsonAny re-exports for callers.
1 parent 7c5f3a5 commit 171b09f

3 files changed

Lines changed: 113 additions & 62 deletions

File tree

‎apps/supervisor/src/env.ts‎

Lines changed: 24 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { randomUUID } from "crypto";
22
import { env as stdEnv } from "std-env";
33
import { z } from "zod";
4-
import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
4+
import { AdditionalEnvVars, BoolEnv, JsonAny, JsonObjectEnv } from "./envUtil.js";
55

66
const Env = z
77
.object({
@@ -94,31 +94,28 @@ const Env = z
9494
KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
9595
KUBERNETES_WORKER_SERVICE_ACCOUNT: z.string().optional(), // Service account for worker pods
9696
KUBERNETES_WORKER_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN: BoolEnv.default(false), // Whether to mount SA token
97-
KUBERNETES_WORKER_POD_ANNOTATIONS: z
98-
.string()
99-
.default("{}")
100-
.transform((v, ctx) => {
101-
try {
102-
const parsed = JSON.parse(v);
103-
if (
104-
typeof parsed !== "object" ||
105-
parsed === null ||
106-
Array.isArray(parsed) ||
107-
Object.values(parsed).some((value) => typeof value !== "string")
108-
) {
109-
throw new Error("expected JSON object of string values");
110-
}
111-
return parsed as Record<string, string>;
112-
} catch (err) {
113-
ctx.addIssue({
114-
code: z.ZodIssueCode.custom,
115-
message: `Invalid KUBERNETES_WORKER_POD_ANNOTATIONS: ${
116-
err instanceof Error ? err.message : String(err)
117-
}`,
118-
});
119-
return z.NEVER;
120-
}
121-
}), // Extra annotations to apply to every worker pod (e.g. for service mesh / cert injection)
97+
// Extra annotations to apply to every worker pod (e.g. for service mesh
98+
// sidecar injection, certificate injection, scheduling hints).
99+
KUBERNETES_WORKER_POD_ANNOTATIONS: JsonObjectEnv("KUBERNETES_WORKER_POD_ANNOTATIONS"),
100+
// Pod-level securityContext applied to every worker pod (V1PodSecurityContext shape).
101+
// Default is empty `{}`, preserving the upstream behavior of not setting
102+
// a pod-level securityContext. Provide a JSON object to enforce e.g.
103+
// `{"runAsNonRoot": true, "runAsUser": 1000, "fsGroup": 1000}`.
104+
// OpenShift and other clusters with arbitrary-UID SCCs typically want
105+
// to leave this empty and let the SCC inject values.
106+
KUBERNETES_WORKER_POD_SECURITY_CONTEXT: JsonObjectEnv("KUBERNETES_WORKER_POD_SECURITY_CONTEXT", {
107+
valueValidator: JsonAny,
108+
}),
109+
// Container-level securityContext applied to the worker container of every
110+
// worker pod (V1SecurityContext shape). Default is empty `{}` (matches
111+
// upstream's previous behavior of not setting a container securityContext).
112+
// Provide a JSON object to enforce e.g.
113+
// `{"runAsNonRoot": true, "runAsUser": 1000, "allowPrivilegeEscalation": false,
114+
// "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}`.
115+
KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT: JsonObjectEnv(
116+
"KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT",
117+
{ valueValidator: JsonAny }
118+
),
122119
KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
123120
KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
124121
KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
@@ -148,16 +145,6 @@ const Env = z
148145

149146
KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB
150147
KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods
151-
152-
// Pod DNS config — override the cluster default ndots to `KUBERNETES_POD_DNS_NDOTS`.
153-
// Default k8s ndots is 5: any name with fewer than 5 dots (e.g. `api.example.com`, 2 dots) is first walked
154-
// through every entry in the cluster search list (`<ns>.svc.cluster.local`, `svc.cluster.local`, `cluster.local`)
155-
// before being tried as-is, turning one resolution into 4+ CoreDNS queries (×2 with A+AAAA).
156-
// Overriding the default can be useful to cut CoreDNS query amplification for external domains.
157-
// Note: before enabling, make sure no code path relies on search-list expansion for names with dots ≥ the value
158-
// set here — those names will now hit their as-is form first and could resolve externally before falling back.
159-
KUBERNETES_POD_DNS_NDOTS_OVERRIDE_ENABLED: BoolEnv.default(false),
160-
KUBERNETES_POD_DNS_NDOTS: z.coerce.number().int().min(1).max(15).default(2),
161148
// Large machine affinity settings - large-* presets prefer a dedicated pool
162149
KUBERNETES_LARGE_MACHINE_AFFINITY_ENABLED: BoolEnv.default(false),
163150
KUBERNETES_LARGE_MACHINE_AFFINITY_POOL_LABEL_KEY: z
@@ -226,9 +213,7 @@ const Env = z
226213
if (!validEffects.includes(effect)) {
227214
ctx.addIssue({
228215
code: z.ZodIssueCode.custom,
229-
message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
230-
", "
231-
)}`,
216+
message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(", ")}`,
232217
});
233218
return z.NEVER;
234219
}

‎apps/supervisor/src/envUtil.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,83 @@ export const AdditionalEnvVars = z.preprocess((val) => {
4545
return undefined;
4646
}
4747
}, z.record(z.string(), z.string()).optional());
48+
49+
/**
50+
* Factory for env vars that hold a JSON object. The default is the empty object,
51+
* so callers can spread the parsed result into Kubernetes manifests without
52+
* branching on undefined.
53+
*
54+
* `valueValidator` constrains the shape of the parsed values:
55+
* - `JsonStringMap` for `Record<string, string>` (e.g. annotations, labels)
56+
* - `JsonAny` for arbitrary nested objects (e.g. `securityContext`)
57+
*
58+
* @example
59+
* KUBERNETES_WORKER_POD_ANNOTATIONS: JsonObjectEnv("KUBERNETES_WORKER_POD_ANNOTATIONS", {
60+
* valueValidator: JsonStringMap,
61+
* }),
62+
*/
63+
export const JsonStringMap = z.record(z.string(), z.string());
64+
export const JsonAny: z.ZodTypeAny = z.lazy(() =>
65+
z.union([
66+
z.string(),
67+
z.number(),
68+
z.boolean(),
69+
z.null(),
70+
z.array(JsonAny),
71+
z.record(z.string(), JsonAny),
72+
])
73+
);
74+
75+
type JsonObjectEnvOpts<TSchema extends z.ZodTypeAny> = {
76+
/**
77+
* Schema applied to each *value* in the parsed object. Defaults to
78+
* `JsonStringMap` (string values).
79+
*/
80+
valueValidator?: TSchema;
81+
};
82+
83+
export const JsonObjectEnv = <TSchema extends z.ZodTypeAny = typeof JsonStringMap>(
84+
envName: string,
85+
opts: JsonObjectEnvOpts<TSchema> = {}
86+
) => {
87+
const valueValidator = (opts.valueValidator ?? JsonStringMap) as TSchema;
88+
89+
return z
90+
.string()
91+
.default("{}")
92+
.transform((raw, ctx) => {
93+
let parsed: unknown;
94+
try {
95+
parsed = JSON.parse(raw);
96+
} catch (e) {
97+
ctx.addIssue({
98+
code: z.ZodIssueCode.custom,
99+
message: `${envName} is not valid JSON: ${e instanceof Error ? e.message : String(e)}`,
100+
});
101+
return z.NEVER;
102+
}
103+
104+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
105+
ctx.addIssue({
106+
code: z.ZodIssueCode.custom,
107+
message: `${envName} must be a JSON object (got ${
108+
Array.isArray(parsed) ? "array" : typeof parsed
109+
})`,
110+
});
111+
return z.NEVER;
112+
}
113+
114+
const validated = z.record(z.string(), valueValidator).safeParse(parsed);
115+
if (!validated.success) {
116+
ctx.addIssue({
117+
code: z.ZodIssueCode.custom,
118+
message: `${envName} has invalid value(s): ${validated.error.message}`,
119+
});
120+
return z.NEVER;
121+
}
122+
123+
return validated.data as z.infer<TSchema> extends z.ZodTypeAny
124+
? Record<string, z.infer<TSchema>>
125+
: Record<string, unknown>;
126+
});
127+
};

‎apps/supervisor/src/workloadManager/kubernetes.ts‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,9 @@ export class KubernetesWorkloadManager implements WorkloadManager {
117117
"app.kubernetes.io/part-of": "trigger-worker",
118118
"app.kubernetes.io/component": "create",
119119
},
120-
annotations: {
121-
...env.KUBERNETES_WORKER_POD_ANNOTATIONS,
122-
},
120+
...(Object.keys(env.KUBERNETES_WORKER_POD_ANNOTATIONS).length > 0
121+
? { annotations: { ...env.KUBERNETES_WORKER_POD_ANNOTATIONS } }
122+
: {}),
123123
},
124124
spec: {
125125
...this.addPlacementTags(this.#defaultPodSpec, opts.placementTags),
@@ -136,14 +136,9 @@ export class KubernetesWorkloadManager implements WorkloadManager {
136136
},
137137
],
138138
resources: this.#getResourcesForMachine(opts.machine),
139-
securityContext: {
140-
runAsNonRoot: true,
141-
runAsUser: 1000,
142-
allowPrivilegeEscalation: false,
143-
capabilities: {
144-
drop: ["ALL"],
145-
},
146-
},
139+
...(Object.keys(env.KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT).length > 0
140+
? { securityContext: env.KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT }
141+
: {}),
147142
env: [
148143
{
149144
name: "TRIGGER_DEQUEUED_AT_MS",
@@ -330,25 +325,16 @@ export class KubernetesWorkloadManager implements WorkloadManager {
330325
...(env.KUBERNETES_WORKER_SERVICE_ACCOUNT
331326
? { serviceAccountName: env.KUBERNETES_WORKER_SERVICE_ACCOUNT }
332327
: {}),
333-
securityContext: {
334-
runAsNonRoot: true,
335-
runAsUser: 1000,
336-
fsGroup: 1000,
337-
},
328+
...(Object.keys(env.KUBERNETES_WORKER_POD_SECURITY_CONTEXT).length > 0
329+
? { securityContext: env.KUBERNETES_WORKER_POD_SECURITY_CONTEXT }
330+
: {}),
338331
...(env.KUBERNETES_WORKER_NODETYPE_LABEL
339332
? {
340333
nodeSelector: {
341334
nodetype: env.KUBERNETES_WORKER_NODETYPE_LABEL,
342335
},
343336
}
344337
: {}),
345-
...(env.KUBERNETES_POD_DNS_NDOTS_OVERRIDE_ENABLED
346-
? {
347-
dnsConfig: {
348-
options: [{ name: "ndots", value: `${env.KUBERNETES_POD_DNS_NDOTS}` }],
349-
},
350-
}
351-
: {}),
352338
};
353339
}
354340

0 commit comments

Comments
 (0)