test(plugins): detect finished-span retention at agent teardown

Add a GC-driven detector that registers every finished span with a
FinalizationRegistry and, at `agent.close()`, forces GC and asserts retention did
not grow with the request count. It catches the failure mode where a captured
async-context frame pins a finished span and its parent chain (#9227): a real
leak retains one span per request and trips the cap, while a clean teardown holds
only the few most-recent traces' roots.

The bound is a small constant, not zero, because a span stores the frame it was
created in (`span._store`), so a trace's spans form a `span -> store -> span`
cycle that stays reachable until a later frame overwrites that async-context
slot. Driving 100 sequential requests through the mock agent collects every root;
a sweep of the http/http2 suites never retained more than two, so the cap is that
measured maximum plus one slot of margin.

The detector is inert without `--expose-gc`, so suites that do not pass the flag
are unaffected.

Refs: #9228

Author: BridgeAR

packages/dd-trace/test/plugins/agent.js

@@ -12,6 +12,7 @@ const semifies = require('semifies')
 
 const { assertObjectContains } = require('../../../../integration-tests/helpers')
 const { storage } = require('../../../datadog-core')
+const spanLeakDetector = require('./span-leak-detector')
 
 // Modules that close over the previous `Config` / `TracerProxy` singletons.
 // Evicted whenever `agent.load`'s gate decides the tracer must rebuild.
@@ -428,6 +429,10 @@ module.exports = {
 
     currentIntegrationName = getCurrentIntegrationName()
 
+    // Track finished spans for the retention assertion in `close()`. Idempotent
+    // and inert without `--expose-gc`, so it is safe to arm on every load.
+    spanLeakDetector.arm()
+
     const tracerConfigJson = JSON.stringify(tracerConfig)
     if (
       !loaded ||
@@ -736,12 +741,17 @@ module.exports = {
 
     tracer.llmobs.disable()
 
-    return /** @type {Promise<void>} */ (new Promise(resolve => {
+    return /** @type {Promise<void>} */ (new Promise((resolve, reject) => {
       this.server.on('close', () => {
         this.server = null
         this.port = null
 
-        resolve()
+        // Expectation callbacks (which can hold spans) were cleared above and
+        // the server is closed, so no legitimate reference to a finished span
+        // remains beyond the last trace's root (see the detector). Assert
+        // retention did not grow with the request count, then resolve. A leak
+        // rejects `close()`, surfacing as a suite teardown failure.
+        spanLeakDetector.assertNoRetainedSpans().then(resolve, reject)
       })
     }))
   },

packages/dd-trace/test/plugins/plugin.spec.js

@@ -77,6 +77,50 @@ describe('Plugin', () => {
     assert.strictEqual(plugin._enabled, true)
   })
 
+  describe('enter', () => {
+    it('should return the store written into storage carrying the span', () => {
+      const parentSpan = { id: 'parent' }
+      const span = { id: 'child' }
+
+      plugin = new GoodPlugin()
+
+      const entered = plugin.enter(span, { span: parentSpan })
+
+      assert.strictEqual(entered.span, span)
+      // The returned object is the one written into storage, so a caller that
+      // holds it can drop the finished span from any frame captured off it.
+      assert.strictEqual(storage('legacy').getStore(), entered)
+    })
+
+    it('should extend the current store when none is passed', () => {
+      const span = { id: 'child' }
+
+      plugin = new GoodPlugin()
+
+      storage('legacy').run({ existing: true, span: undefined }, () => {
+        const entered = plugin.enter(span)
+
+        assert.deepStrictEqual(entered, { existing: true, span })
+        assert.strictEqual(storage('legacy').getStore(), entered)
+      })
+    })
+
+    it('should let a caller drop the finished span by nulling the returned store', () => {
+      const span = { id: 'child' }
+
+      plugin = new GoodPlugin()
+      const entered = plugin.enter(span, { span: undefined })
+
+      assert.strictEqual(entered.span, span)
+
+      // The release contract callers rely on at request finish: nulling the
+      // returned store's span means a frame that captured it stops pinning it.
+      entered.span = null
+
+      assert.strictEqual(entered.span, null)
+    })
+  })
+
   it('should run binding transforms with an undefined current store', () => {
     class TestPlugin extends Plugin {
       static id = 'test'

packages/dd-trace/test/plugins/span-leak-detector.js

@@ -0,0 +1,154 @@
+'use strict'
+
+const assert = require('node:assert')
+
+const { channel } = require('dc-polyfill')
+
+const finishCh = channel('dd-trace:span:finish')
+
+// A finished span's retention must not scale with the number of requests. The
+// classic violation is an async-context frame that captured a `{ ...store, span }`
+// object (see `Plugin#enter`): the frame outlives the request via any
+// never-released async resource and pins the finished span and its whole parent
+// chain, so retained spans grow one-per-request until the process OOMs
+// (DataDog/dd-trace-js#9227).
+//
+// This detector rides along on every suite that runs with `--expose-gc`: it
+// registers each finished span with a FinalizationRegistry and, at agent
+// teardown, forces GC and asserts retention did not grow with the request count.
+// It is a no-op without `--expose-gc`, so suites that do not pass the flag are
+// unaffected.
+//
+// Why the bound is a small constant, not zero: a span stores the async-context
+// frame it was created in (`span._store = storage('legacy').getHandle()`, see
+// `opentracing/span.js`). That frame is `{ ...parent, span }`, so a trace's spans
+// form a reference cycle (`span._store -> store -> span`) that stays reachable
+// until a *later* frame overwrites that async-context slot. At teardown the few
+// most-recently-created traces' frames have not been overwritten yet, so their
+// roots can still be alive. This is bounded by recency, not by request count:
+// driving 100 sequential requests through the mock-agent harness collects every
+// root (`retained === 0`), and a wide sweep of the http/http2 suites never
+// retained more than two (only the 1-2 requests a test makes right before its
+// teardown). A real leak like #9227 pins one span per request for the life of the
+// process, so `retained` climbs into the dozens or hundreds and trips the cap.
+
+const gc = typeof global.gc === 'function' ? global.gc : undefined
+
+/**
+ * Number of forced-GC + drain cycles before giving up. FinalizationRegistry
+ * callbacks are scheduled after collection and V8 may need more than one pass,
+ * so poll rather than assert after a single GC.
+ */
+const MAX_GC_CYCLES = 10
+
+/**
+ * Upper bound on finished spans that may still be reachable at teardown without
+ * indicating a leak. Benign retention is the handful of most-recent traces whose
+ * async-context frames have not been overwritten yet (see above); a sweep of the
+ * http/http2 suites topped out at two, so this is that measured maximum plus one
+ * slot of margin. It is an absolute cap, not a per-request allowance: a #9227-style
+ * leak retains one span per request and blows past it by orders of magnitude, so
+ * the margin costs no detection power.
+ */
+const BASELINE_RETAINED = 3
+
+class SpanLeakDetector {
+  #registry
+  #trackedCount = 0
+  #collectedCount = 0
+  #armed = false
+  #onFinish = span => {
+    this.#trackedCount++
+    this.#registry.register(span)
+  }
+
+  /**
+   * Whether span-retention checking is possible in this process. Requires
+   * `--expose-gc`; the detector is inert otherwise.
+   *
+   * @returns {boolean}
+   */
+  get enabled () {
+    return gc !== undefined
+  }
+
+  /**
+   * Start tracking finished spans. Idempotent so repeated `agent.load` calls in
+   * one suite do not double-subscribe (which would inflate the tracked count and
+   * leak the finish handler across load/close cycles).
+   *
+   * @returns {void}
+   */
+  arm () {
+    if (!this.enabled || this.#armed) return
+    // Lazily create the registry so processes without `--expose-gc` never build
+    // one. The callback only counts; it must not retain the span.
+    this.#registry ??= new FinalizationRegistry(() => {
+      this.#collectedCount++
+    })
+    finishCh.subscribe(this.#onFinish)
+    this.#armed = true
+  }
+
+  /**
+   * Force GC and assert that finished-span retention has not grown with the
+   * request count. Call once the tracer has flushed and no test-owned reference
+   * to a span remains (i.e. at agent teardown, after expectation callbacks are
+   * cleared). Up to {@link BASELINE_RETAINED} spans may survive (the few
+   * most-recent traces' roots, still held by their `span._store` cycle); more
+   * than that means a tracer-owned structure is pinning finished spans
+   * one-per-request. Resets counters so the next suite starts clean. No-op when
+   * the detector is inert or nothing was tracked.
+   *
+   * @returns {Promise<void>}
+   */
+  async assertNoRetainedSpans () {
+    if (!this.#armed) return
+
+    finishCh.unsubscribe(this.#onFinish)
+    this.#armed = false
+
+    if (this.#trackedCount === 0) {
+      this.#reset()
+      return
+    }
+
+    // Let any pending flush / microtasks settle, then force GC and give the
+    // registry callbacks a turn. Repeat until retention is within the baseline or
+    // the cycle budget is spent (polling because FinalizationRegistry callbacks
+    // run after collection and V8 may need more than one pass).
+    for (
+      let cycle = 0;
+      cycle < MAX_GC_CYCLES && this.#trackedCount - this.#collectedCount > BASELINE_RETAINED;
+      cycle++
+    ) {
+      await new Promise(resolve => setImmediate(resolve))
+      gc()
+    }
+    await new Promise(resolve => setImmediate(resolve))
+
+    const retained = this.#trackedCount - this.#collectedCount
+    const tracked = this.#trackedCount
+    this.#reset()
+
+    assert.ok(
+      retained <= BASELINE_RETAINED,
+      `${retained} of ${tracked} finished spans were still reachable after flush + GC ` +
+      `(at most ${BASELINE_RETAINED} may survive as recently-created traces whose async-context ` +
+      'frames have not been overwritten yet). Retention is growing with the request count — a ' +
+      'tracer-owned structure is pinning finished spans, commonly an async-context frame that ' +
+      'captured a `{ ...store, span }` object without a matching `releaseSpan` ' +
+      '(see DataDog/dd-trace-js#9227).'
+    )
+  }
+
+  #reset () {
+    this.#trackedCount = 0
+    this.#collectedCount = 0
+  }
+}
+
+module.exports = new SpanLeakDetector()
+// Exposed for the detector's own spec, which asserts both the retained and the
+// freed outcome against fresh instances without a real agent teardown.
+module.exports.SpanLeakDetector = SpanLeakDetector

packages/dd-trace/test/plugins/span-leak-detector.spec.js

@@ -0,0 +1,123 @@
+'use strict'
+
+const assert = require('node:assert')
+
+const { describe, it } = require('mocha')
+const { channel } = require('dc-polyfill')
+
+const { SpanLeakDetector } = require('./span-leak-detector')
+
+const finishCh = channel('dd-trace:span:finish')
+
+if (typeof global.gc !== 'function') {
+  throw new Error('span-leak-detector.spec.js requires --expose-gc')
+}
+
+// Pins the retention contract the agent-teardown check relies on: retention that
+// grows with the request count must be reported as a leak, while the small,
+// recency-bounded baseline (the few most-recent traces whose async-context frames
+// have not been overwritten yet) must pass. The detector tolerates up to
+// BASELINE_RETAINED (3) survivors — a measured maximum of two across the http
+// suites plus one slot of margin — so the boundary cases below pin 3 (accepted)
+// and 4 (the first rejected). Uses fresh detector instances (not the agent
+// singleton) so every outcome can be asserted without a real teardown.
+
+const BASELINE_RETAINED = 3
+
+async function forceGc () {
+  for (let i = 0; i < 10; i++) {
+    await new Promise(resolve => setImmediate(resolve))
+    global.gc()
+  }
+}
+
+/**
+ * Publish `count` finish events for spans held by strong references, mirroring
+ * finished spans pinned by a never-released async-context frame. The references
+ * are returned so the caller keeps them live across the assertion.
+ *
+ * @param {number} count
+ * @returns {Array<{ _name: string }>}
+ */
+function publishRetained (count) {
+  const spans = []
+  for (let i = 0; i < count; i++) {
+    const span = { _name: `leaked.span.${i}` }
+    spans.push(span)
+    finishCh.publish(span)
+  }
+  return spans
+}
+
+describe('span-leak-detector', () => {
+  it('tolerates retention at the baseline (the recency-bounded roots)', async () => {
+    const detector = new SpanLeakDetector()
+    detector.arm()
+
+    // Exactly BASELINE_RETAINED survivors mirror the last few traces' roots still
+    // held by their `span._store` cycle at teardown; bounded, not a leak.
+    const baselineSpans = publishRetained(BASELINE_RETAINED)
+
+    await detector.assertNoRetainedSpans()
+    assert.strictEqual(baselineSpans.length, BASELINE_RETAINED)
+  })
+
+  it('reports a leak when retention exceeds the baseline by one', async () => {
+    const detector = new SpanLeakDetector()
+    detector.arm()
+
+    // One past the baseline is the smallest retention that indicates growth with
+    // the request count rather than the bounded recency window.
+    const retainedSpans = publishRetained(BASELINE_RETAINED + 1)
+
+    await assert.rejects(
+      () => detector.assertNoRetainedSpans(),
+      new RegExp(`${BASELINE_RETAINED + 1} of ${BASELINE_RETAINED + 1} finished spans were still reachable`)
+    )
+
+    // References stay live through the assertion, mirroring a captured frame.
+    assert.strictEqual(retainedSpans.length, BASELINE_RETAINED + 1)
+  })
+
+  it('reports no retention when nothing holds the finished span', async () => {
+    const detector = new SpanLeakDetector()
+    detector.arm()
+
+    // Publish a span held by nothing so no reference survives to the assertion.
+    ;(() => finishCh.publish({ _name: 'collectable.span' }))()
+
+    await forceGc()
+    await detector.assertNoRetainedSpans()
+  })
+
+  it('is a no-op when armed but nothing was tracked', async () => {
+    const detector = new SpanLeakDetector()
+    detector.arm()
+
+    await detector.assertNoRetainedSpans()
+  })
+
+  it('is a no-op when never armed', async () => {
+    const detector = new SpanLeakDetector()
+
+    await detector.assertNoRetainedSpans()
+  })
+
+  it('does not double-count finished spans when armed twice', async () => {
+    const detector = new SpanLeakDetector()
+    detector.arm()
+    detector.arm()
+
+    // Spans must be counted once each, not once per arm() call — otherwise a
+    // repeated agent.load in one suite would inflate the count and leak the finish
+    // handler. Publish one past the baseline so the (correct) count still trips
+    // the assertion; a double-count would report twice as many.
+    const retainedSpans = publishRetained(BASELINE_RETAINED + 1)
+
+    await assert.rejects(
+      () => detector.assertNoRetainedSpans(),
+      new RegExp(`${BASELINE_RETAINED + 1} of ${BASELINE_RETAINED + 1} finished spans were still reachable`)
+    )
+    assert.strictEqual(retainedSpans.length, BASELINE_RETAINED + 1)
+  })
+})