fix(discord): ignore Jaeger file cache in memory alerts (#94)

Author: patroza

apps/discord-bot/src/features/Alerts.test.ts

@@ -5,6 +5,8 @@ import {
   classifySessionLastError,
   formatAlertCause,
   isExpectedSessionLastError,
+  memoryPolicyForProcess,
+  parseProcMemoryStatus,
   selectSessionErrorsForAlert,
   sessionErrorAlertKey,
   trackSustainedHotProcesses,
@@ -20,6 +22,8 @@ const SUSTAINED_TICKS = 5;
 const proc = (over: Partial<ProcInfo> & { pid: number }): ProcInfo => ({
   pid: over.pid,
   rssMb: over.rssMb ?? 100,
+  rssAnonMb: over.rssAnonMb ?? 50,
+  rssFileMb: over.rssFileMb ?? 50,
   cpuSeconds: over.cpuSeconds ?? 0,
   cmd: over.cmd ?? `/bin/proc-${over.pid}`,
   label: over.label ?? `proc-${over.pid}`,
@@ -42,6 +46,7 @@ function run(
       nowMs: startMs + index * TICK_MS,
       cpuPercentThreshold: CPU_THRESHOLD,
       rssMbThreshold: RSS_THRESHOLD,
+      memoryPolicyFor: (process) => memoryPolicyForProcess(process, RSS_THRESHOLD),
       sustainedTicks: SUSTAINED_TICKS,
     });
     state = result.next;
@@ -126,6 +131,20 @@ describe("session last_error alert classification", () => {
 });
 
 describe("trackSustainedHotProcesses", () => {
+  it("parses total, anonymous, and file-backed RSS from proc status", () => {
+    expect(
+      parseProcMemoryStatus(`
+VmRSS:       1638400 kB
+RssAnon:      921600 kB
+RssFile:      716800 kB
+`),
+    ).toEqual({
+      rssMb: 1_600,
+      rssAnonMb: 900,
+      rssFileMb: 700,
+    });
+  });
+
   it("does not alert on a long-lived but idle process", () => {
     // The reported bug: a process with lots of cumulative CPU time that now barely
     // moves (a few seconds every tick) must never alert. +2s of CPU per 60s tick
@@ -173,9 +192,63 @@ describe("trackSustainedHotProcesses", () => {
     const hot = run(ticks).hot;
     expect(hot).toHaveLength(1);
     expect(hot[0]!.rssMb).toBe(900);
+    expect(hot[0]!.memoryKind).toBe("rss");
     expect(hot[0]!.cpuPercent).toBe(0);
   });
 
+  it("ignores Jaeger file cache below the anonymous-memory threshold", () => {
+    const ticks = Array.from({ length: SUSTAINED_TICKS + 1 }, () => [
+      proc({
+        pid: 950,
+        rssMb: 3_000,
+        rssAnonMb: 900,
+        rssFileMb: 2_100,
+        cmd: "/cmd/jaeger/jaeger-linux --config=/etc/jaeger/config.yaml",
+      }),
+    ]);
+
+    expect(run(ticks).hot).toEqual([]);
+  });
+
+  it("alerts once Jaeger sustains 2 GiB of anonymous memory", () => {
+    const ticks = Array.from({ length: SUSTAINED_TICKS + 1 }, () => [
+      proc({
+        pid: 950,
+        rssMb: 3_000,
+        rssAnonMb: 2 * 1024,
+        rssFileMb: 952,
+        cmd: "/cmd/jaeger/jaeger-linux --config=/etc/jaeger/config.yaml",
+      }),
+    ]);
+
+    expect(run(ticks).hot).toEqual([
+      expect.objectContaining({
+        pid: 950,
+        memoryKind: "anonymous",
+        memoryValueMb: 2 * 1024,
+        memoryThresholdMb: 2 * 1024,
+      }),
+    ]);
+  });
+
+  it("falls back to total RSS when the kernel omits the Jaeger RSS breakdown", () => {
+    expect(
+      memoryPolicyForProcess(
+        proc({
+          pid: 950,
+          rssMb: 2 * 1024,
+          rssAnonMb: 0,
+          rssFileMb: 0,
+          cmd: "/cmd/jaeger/jaeger-linux",
+        }),
+      ),
+    ).toEqual({
+      valueMb: 2 * 1024,
+      thresholdMb: 2 * 1024,
+      kind: "rss",
+    });
+  });
+
   it("treats a reused pid as a new process and resets its streak", () => {
     const busy = Array.from({ length: 4 }, (_unused, i) => [
       proc({ pid: 900, cpuSeconds: i * 60 }),

apps/discord-bot/src/features/Alerts.ts

@@ -38,10 +38,11 @@ const DISK_FREE_MIN_GB = 2;
 const SENTRY_RSS_ALERT_MB = 512;
 const SENTRY_COUNT_ALERT = 2;
 const STUCK_RSS_ALERT_MB = 768;
+const JAEGER_ANON_ALERT_MB = 2 * 1024;
 /**
- * A process is "hot" when it holds ≥STUCK_RSS_ALERT_MB of RSS or averages
- * ≥SUSTAINED_CPU_PERCENT of a core, and it only alerts once it has stayed hot
- * for SUSTAINED_TICKS consecutive ticks.
+ * A process is "hot" when it exceeds its memory policy or averages at least
+ * SUSTAINED_CPU_PERCENT of a core, and it only alerts once it has stayed hot for
+ * SUSTAINED_TICKS consecutive ticks.
  *
  * This measures a *rate* (Δcpu / Δwall between ticks), not cumulative CPU time:
  * a long-lived-but-idle process (e.g. one that gathered 200s of CPU over hours
@@ -77,11 +78,38 @@ const RUNAWAY_PATTERNS: ReadonlyArray<{
 export interface ProcInfo {
   readonly pid: number;
   readonly rssMb: number;
+  readonly rssAnonMb: number;
+  readonly rssFileMb: number;
   readonly cpuSeconds: number;
   readonly cmd: string;
   readonly label: string;
 }
 
+export interface ProcMemoryPolicy {
+  readonly valueMb: number;
+  readonly thresholdMb: number;
+  readonly kind: "anonymous" | "rss";
+}
+
+export function memoryPolicyForProcess(
+  proc: ProcInfo,
+  defaultRssMbThreshold = STUCK_RSS_ALERT_MB,
+): ProcMemoryPolicy {
+  if (/(?:^|[/\s])jaeger(?:$|[/\s-])/i.test(proc.cmd)) {
+    const hasRssBreakdown = proc.rssAnonMb > 0 || proc.rssFileMb > 0;
+    return {
+      valueMb: hasRssBreakdown ? proc.rssAnonMb : proc.rssMb,
+      thresholdMb: JAEGER_ANON_ALERT_MB,
+      kind: hasRssBreakdown ? "anonymous" : "rss",
+    };
+  }
+  return {
+    valueMb: proc.rssMb,
+    thresholdMb: defaultRssMbThreshold,
+    kind: "rss",
+  };
+}
+
 /** Per-process tracker state carried between ticks to derive a CPU rate. */
 export interface ProcSustainState {
   readonly cpuSeconds: number;
@@ -96,6 +124,11 @@ export interface ProcSustainState {
 export interface SustainedHotProcess {
   readonly pid: number;
   readonly rssMb: number;
+  readonly rssAnonMb: number;
+  readonly rssFileMb: number;
+  readonly memoryValueMb: number;
+  readonly memoryThresholdMb: number;
+  readonly memoryKind: ProcMemoryPolicy["kind"];
   /** Average CPU over the last tick gap, as percent of a single core. */
   readonly cpuPercent: number;
   /** How long it has been continuously hot. */
@@ -116,6 +149,7 @@ export function trackSustainedHotProcesses(input: {
   readonly nowMs: number;
   readonly cpuPercentThreshold: number;
   readonly rssMbThreshold: number;
+  readonly memoryPolicyFor?: (proc: ProcInfo) => ProcMemoryPolicy;
   readonly sustainedTicks: number;
 }): {
   readonly next: Map<number, ProcSustainState>;
@@ -136,9 +170,14 @@ export function trackSustainedHotProcesses(input: {
         ? (Math.max(0, proc.cpuSeconds - previous.cpuSeconds) / (elapsedMs / 1_000)) * 100
         : null;
 
+    const memoryPolicy = input.memoryPolicyFor?.(proc) ?? {
+      valueMb: proc.rssMb,
+      thresholdMb: input.rssMbThreshold,
+      kind: "rss",
+    };
     const isHot =
       (cpuPercent !== null && cpuPercent >= input.cpuPercentThreshold) ||
-      proc.rssMb >= input.rssMbThreshold;
+      memoryPolicy.valueMb >= memoryPolicy.thresholdMb;
     const hotTicks = isHot ? (previous?.hotTicks ?? 0) + 1 : 0;
     const hotSinceMs = isHot
       ? previous?.hotTicks
@@ -157,14 +196,19 @@ export function trackSustainedHotProcesses(input: {
       hot.push({
         pid: proc.pid,
         rssMb: proc.rssMb,
+        rssAnonMb: proc.rssAnonMb,
+        rssFileMb: proc.rssFileMb,
+        memoryValueMb: memoryPolicy.valueMb,
+        memoryThresholdMb: memoryPolicy.thresholdMb,
+        memoryKind: memoryPolicy.kind,
         cpuPercent: cpuPercent ?? 0,
         sustainedMs: input.nowMs - hotSinceMs,
         label: proc.label,
       });
     }
   }
 
-  hot.sort((a, b) => b.cpuPercent - a.cpuPercent || b.rssMb - a.rssMb);
+  hot.sort((a, b) => b.cpuPercent - a.cpuPercent || b.memoryValueMb - a.memoryValueMb);
   return { next, hot: hot.slice(0, 8) };
 }
 
@@ -354,13 +398,32 @@ function readDisk(path: string): DiskInfo | null {
   }
 }
 
-function readRssMb(pid: number): number {
+export function parseProcMemoryStatus(status: string): {
+  readonly rssMb: number;
+  readonly rssAnonMb: number;
+  readonly rssFileMb: number;
+} {
+  const readMb = (field: string) => {
+    const match = new RegExp(`^${field}:\\s+(\\d+)\\s+kB`, "m").exec(status);
+    return match ? Number(match[1]) / 1024 : 0;
+  };
+  return {
+    rssMb: readMb("VmRSS"),
+    rssAnonMb: readMb("RssAnon"),
+    rssFileMb: readMb("RssFile"),
+  };
+}
+
+function readProcMemory(pid: number): {
+  readonly rssMb: number;
+  readonly rssAnonMb: number;
+  readonly rssFileMb: number;
+} {
   try {
     const status = NodeFS.readFileSync(`/proc/${pid}/status`, "utf8");
-    const match = /^VmRSS:\s+(\d+)\s+kB/m.exec(status);
-    return match ? Number(match[1]) / 1024 : 0;
+    return parseProcMemoryStatus(status);
   } catch {
-    return 0;
+    return { rssMb: 0, rssAnonMb: 0, rssFileMb: 0 };
   }
 }
 
@@ -414,9 +477,10 @@ function listProcesses(): ReadonlyArray<ProcInfo> {
     if (pid === process.pid) continue;
     const cmd = readCmdline(pid);
     if (cmd === "") continue;
+    const memory = readProcMemory(pid);
     out.push({
       pid,
-      rssMb: readRssMb(pid),
+      ...memory,
       cpuSeconds: readCpuSeconds(pid),
       cmd,
       label: shortCmd(cmd),
@@ -456,6 +520,7 @@ function listFatProcesses(
     nowMs,
     cpuPercentThreshold: SUSTAINED_CPU_PERCENT,
     rssMbThreshold: STUCK_RSS_ALERT_MB,
+    memoryPolicyFor: (proc) => memoryPolicyForProcess(proc, STUCK_RSS_ALERT_MB),
     sustainedTicks: SUSTAINED_TICKS,
   });
   sustainState = next;
@@ -803,13 +868,20 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) =>
           yield* postAlert(
             "stuck-proc",
             [
-              "**Sustained high RSS / CPU process(es)**",
-              ...fatNonRunaway.map(
-                (p) =>
-                  `• pid=${p.pid} rss=${p.rssMb.toFixed(0)}MiB cpu≈${p.cpuPercent.toFixed(0)}% ` +
-                  `for ${Math.round(p.sustainedMs / 60_000)}m ${p.label}`,
-              ),
-              `_Sustained ≥${SUSTAINED_TICKS} ticks with RSS≥${STUCK_RSS_ALERT_MB}MiB or CPU≥${SUSTAINED_CPU_PERCENT}% of a core (not auto-killed)._`,
+              "**Sustained high memory / CPU process(es)**",
+              ...fatNonRunaway.map((p) => {
+                const memory =
+                  p.memoryKind === "anonymous"
+                    ? `anon=${p.rssAnonMb.toFixed(0)}MiB file-cache=${p.rssFileMb.toFixed(0)}MiB rss=${p.rssMb.toFixed(0)}MiB`
+                    : `rss=${p.rssMb.toFixed(0)}MiB`;
+                return (
+                  `• pid=${p.pid} ${memory} ` +
+                  `(memory alert ${p.memoryKind}≥${p.memoryThresholdMb.toFixed(0)}MiB) ` +
+                  `cpu≈${p.cpuPercent.toFixed(0)}% ` +
+                  `for ${Math.round(p.sustainedMs / 60_000)}m ${p.label}`
+                );
+              }),
+              `_Sustained ≥${SUSTAINED_TICKS} ticks with process-specific high memory or CPU≥${SUSTAINED_CPU_PERCENT}% of a core (not auto-killed)._`,
             ].join("\n"),
           );
         }