fix: honor ignore on enter for toolsets (#2011)

Co-authored-by: rosetta-livekit-bot[bot] <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com>
Co-authored-by: Toubat <brian.yin@livekit.io>

Author: rosetta-livekit-bot[bot]

.changeset/honor-ignore-on-enter-toolsets.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents': patch
+---
+
+Honor `ToolFlag.IGNORE_ON_ENTER` for tools nested inside `Toolset`s.

agents/src/beta/tools/end_call.ts

@@ -8,6 +8,7 @@ import {
   RealtimeModel,
   type ToolCalledEvent,
   type ToolCompletedEvent,
+  ToolFlag,
   Toolset,
   tool,
 } from '../../llm/index.js';
@@ -68,6 +69,8 @@ export type EndCallToolOptions<UserData = UnknownUserData> = {
   deleteRoom?: boolean;
   /** Tool output to the LLM for generating the tool response. */
   endInstructions?: string | null;
+  /** Hide the tool during onEnter so the model can't end the call while greeting. */
+  ignoreOnEnter?: boolean;
   /** Callback to call when the tool is called. */
   onToolCalled?: (event: EndCallToolCalledEvent<UserData>) => Promise<void> | void;
   /** Callback to call when the tool is completed. */
@@ -81,6 +84,7 @@ export function createEndCallTool<UserData = UnknownUserData>({
   extraDescription = '',
   deleteRoom = true,
   endInstructions = 'say goodbye to the user',
+  ignoreOnEnter = false,
   onToolCalled,
   onToolCompleted,
 }: EndCallToolOptions<UserData> = {}): Toolset {
@@ -116,6 +120,7 @@ export function createEndCallTool<UserData = UnknownUserData>({
       tool<UserData>({
         name: 'end_call',
         description: `${END_CALL_DESCRIPTION}\n${extraDescription}`,
+        flags: ignoreOnEnter ? ToolFlag.IGNORE_ON_ENTER : ToolFlag.NONE,
         execute: async (_args, { ctx, abortSignal }) => {
           log().debug('end_call tool called');
           const session = ctx.session;

agents/src/llm/tool_context.test.ts

@@ -5,6 +5,7 @@ import { describe, expect, it } from 'vitest';
 import { z } from 'zod';
 import * as z3 from 'zod/v3';
 import * as z4 from 'zod/v4';
+import { createEndCallTool } from '../beta/tools/end_call.js';
 import { voice } from '../index.js';
 import {
   CONFIRM_DUPLICATE_PARAM,
@@ -15,6 +16,7 @@ import {
   type ToolOptions,
   Toolset,
   type ToolsetContext,
+  isFunctionTool,
   tool,
 } from './tool_context.js';
 import { createToolOptions, oaiParams } from './utils.js';
@@ -701,6 +703,32 @@ describe('ToolContext', () => {
     expect(ctx.getFunctionTool('b')).toBe(b);
   });
 
+  it('_exclude hides a toolset member but keeps the toolset', () => {
+    const a = makeFn('a');
+    const b = makeFn('b');
+    const direct = makeFn('direct');
+    const ts = new Toolset({ id: 'set', tools: [a, b] });
+    const ctx = new ToolContext([ts, direct]);
+
+    ctx._exclude([a]);
+
+    expect(ctx.getFunctionTool('a')).toBeUndefined();
+    expect(ctx.flatten()).not.toContain(a);
+    expect(ctx.getFunctionTool('b')).toBe(b);
+    expect(ctx.getFunctionTool('direct')).toBe(direct);
+    expect(ctx.toolsets).toEqual([ts]);
+  });
+
+  it('_exclude with no tools is a no-op', () => {
+    const a = makeFn('a');
+    const b = makeFn('b');
+    const ctx = new ToolContext([a, b]);
+
+    ctx._exclude([]);
+
+    expect(Object.keys(ctx.functionTools).sort()).toEqual(['a', 'b']);
+  });
+
   it('copy() yields an independent context with the same tools', () => {
     const a = makeFn('a');
     const ctx = new ToolContext([a]);
@@ -768,6 +796,22 @@ describe('ToolContext', () => {
   });
 });
 
+describe('createEndCallTool', () => {
+  it('sets IGNORE_ON_ENTER only when requested', () => {
+    const [defaultTool] = createEndCallTool().tools;
+    expect(defaultTool).toBeDefined();
+    expect(isFunctionTool(defaultTool)).toBe(true);
+    expect(isFunctionTool(defaultTool) && defaultTool.flags & ToolFlag.IGNORE_ON_ENTER).toBe(0);
+
+    const [ignoredTool] = createEndCallTool({ ignoreOnEnter: true }).tools;
+    expect(ignoredTool).toBeDefined();
+    expect(isFunctionTool(ignoredTool)).toBe(true);
+    expect(
+      isFunctionTool(ignoredTool) && ignoredTool.flags & ToolFlag.IGNORE_ON_ENTER,
+    ).toBeTruthy();
+  });
+});
+
 describe('Toolset', () => {
   const makeFn = (name: string) =>
     tool({

agents/src/llm/tool_context.ts

@@ -519,14 +519,23 @@ export class ToolContext<UserData = UnknownUserData> {
   }
 
   updateTools(tools: ToolContextInit<UserData>): void {
+    this._updateTools(tools);
+  }
+
+  private _updateTools(tools: ToolContextInit<UserData>, exclude: readonly Tool[] = []): void {
     const normalizedTools = normalizeToolContextInit(tools);
+    const excludedTools = new Set<unknown>(exclude);
     this._tools = normalizedTools;
     this._functionToolsMap = new Map();
     this._providerTools = [];
     this._toolsets = [];
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any -- accepts any tool shape
     const addTool = (tool: any): void => {
+      if (excludedTools.has(tool)) {
+        return;
+      }
+
       if (isToolset(tool)) {
         for (const inner of tool.tools) {
           addTool(inner);
@@ -560,6 +569,42 @@ export class ToolContext<UserData = UnknownUserData> {
     }
   }
 
+  private _syncFlattened(tools: readonly Tool[]): void {
+    const current = this.flatten();
+    const currentTools = new Set(current);
+    const nextTools = new Set(tools);
+    if (currentTools.size === nextTools.size && current.every((tool) => nextTools.has(tool))) {
+      return;
+    }
+
+    const added: Extract<ToolContextEntry<UserData>, Tool>[] = tools
+      .filter((tool) => !currentTools.has(tool))
+      .filter(
+        (tool): tool is Extract<ToolContextEntry<UserData>, Tool> =>
+          isFunctionTool(tool) || isProviderTool(tool),
+      );
+    const removed = current.filter((tool) => !nextTools.has(tool));
+    const structured = this._tools.filter((tool) => !removed.includes(tool as Tool));
+    this._updateTools([...structured, ...added], removed);
+  }
+
+  /** Hide tools from the callable set while keeping their toolsets intact. @internal */
+  _exclude(tools: readonly Tool[]): void {
+    if (tools.length === 0) {
+      return;
+    }
+
+    const excludedTools = new Set(tools);
+    this._syncFlattened(this.flatten().filter((tool) => !excludedTools.has(tool)));
+  }
+
+  /** Return a copy containing only flattened callable/provider entries. @internal */
+  _flattenedCopy(): ToolContext<UserData> {
+    const copy = ToolContext.empty<UserData>();
+    copy._syncFlattened(this.flatten());
+    return copy;
+  }
+
   copy(): ToolContext<UserData> {
     return new ToolContext<UserData>([...this._tools]);
   }

agents/src/voice/agent_activity.test.ts

@@ -21,7 +21,7 @@ import { LLM, type LLMStream } from '../llm/llm.js';
 import { type Tool, ToolContext, ToolFlag, Toolset, tool } from '../llm/tool_context.js';
 import { Future, Task } from '../utils.js';
 import { _getActivityTaskInfo } from './agent.js';
-import { AgentActivity } from './agent_activity.js';
+import { AgentActivity, onEnterStorage } from './agent_activity.js';
 import type { PreemptiveGenerationInfo } from './audio_recognition.js';
 import type { AgentSessionEventTypes, UserInputTranscribedEvent } from './events.js';
 import { SpeechHandle } from './speech_handle.js';
@@ -582,6 +582,77 @@ describe('AgentActivity - onPreemptiveGeneration guards', () => {
   });
 });
 
+describe('AgentActivity - onEnter ignored tools', () => {
+  const makeFn = (name: string, flags = ToolFlag.NONE) =>
+    tool({ name, description: `${name} tool`, flags, execute: async () => name });
+
+  function buildIgnoredToolsActivity() {
+    const activity = Object.create(AgentActivity.prototype) as AgentActivity;
+    Object.assign(activity, {
+      agent: {},
+      agentSession: {},
+    });
+    return activity;
+  }
+
+  it('returns bare and toolset-nested IGNORE_ON_ENTER tools only inside this onEnter', () => {
+    const endCall = makeFn('end_call', ToolFlag.IGNORE_ON_ENTER);
+    const keep = makeFn('keep');
+    const bareIgnored = makeFn('bare_ignored', ToolFlag.IGNORE_ON_ENTER);
+    const bareKeep = makeFn('bare_keep');
+    const toolset = new Toolset({ id: 'ts', tools: [endCall, keep] });
+    const toolCtx = new ToolContext([toolset, bareIgnored, bareKeep]);
+    const activity = buildIgnoredToolsActivity();
+
+    expect(activity._onEnterIgnoredTools(toolCtx)).toEqual([]);
+
+    onEnterStorage.run({ session: activity.agentSession, agent: activity.agent }, () => {
+      expect(
+        activity
+          ._onEnterIgnoredTools(toolCtx)
+          .map((t) => t.id)
+          .sort(),
+      ).toEqual(['bare_ignored', 'end_call']);
+    });
+
+    onEnterStorage.run({ session: activity.agentSession, agent: {} as never }, () => {
+      expect(activity._onEnterIgnoredTools(toolCtx)).toEqual([]);
+    });
+  });
+
+  it('hides ignored bare and toolset tools while preserving normal tools for nested replies', async () => {
+    const endCall = makeFn('end_call', ToolFlag.IGNORE_ON_ENTER);
+    const keep = makeFn('keep');
+    const bareIgnored = makeFn('bare_ignored', ToolFlag.IGNORE_ON_ENTER);
+    const toolset = new Toolset({ id: 'ts', tools: [endCall, keep] });
+    const activity = buildIgnoredToolsActivity();
+
+    await onEnterStorage.run(
+      { session: activity.agentSession, agent: activity.agent },
+      async () => {
+        const greetingTools = new ToolContext([toolset, bareIgnored]);
+        greetingTools._exclude(activity._onEnterIgnoredTools(greetingTools));
+        expect(greetingTools.flatten().map((t) => t.id)).toEqual(['keep']);
+        expect(greetingTools.toolsets).toEqual([toolset]);
+
+        await Promise.resolve();
+
+        const toolReplyTools = new ToolContext([toolset, bareIgnored]);
+        toolReplyTools._exclude(activity._onEnterIgnoredTools(toolReplyTools));
+        expect(toolReplyTools.flatten().map((t) => t.id)).toEqual(['keep']);
+      },
+    );
+
+    const restoredTools = new ToolContext([toolset, bareIgnored]);
+    expect(
+      restoredTools
+        .flatten()
+        .map((t) => t.id)
+        .sort(),
+    ).toEqual(['bare_ignored', 'end_call', 'keep']);
+  });
+});
+
 /**
  * Regression test for the dynamic-toolset push path.
  *

agents/src/voice/agent_activity.ts

@@ -41,13 +41,13 @@ import {
   RealtimeModel,
   type RealtimeModelError,
   type RealtimeSession,
+  type Tool,
   type ToolChoice,
   ToolContext,
   type ToolContextEntry,
   type ToolContextLike,
   ToolFlag,
   isFunctionTool,
-  isToolset,
   toToolContext,
 } from '../llm/index.js';
 import type { LLMError } from '../llm/llm.js';
@@ -2136,19 +2136,8 @@ export class AgentActivity implements RecognitionHooks {
         instructions = concatInstructions(this.agent.instructions, '\n', instructions);
       }
 
-      // Filter out tools with IGNORE_ON_ENTER flag when generateReply is called inside onEnter
-      const onEnterData = onEnterStorage.getStore();
-      const shouldFilterTools =
-        onEnterData?.agent === this.agent && onEnterData?.session === this.agentSession;
-
-      const tools: ToolContext = shouldFilterTools
-        ? new ToolContext(
-            this.tools.tools.filter((t): boolean => {
-              if (isToolset(t) || !isFunctionTool(t)) return true;
-              return !(t.flags & ToolFlag.IGNORE_ON_ENTER);
-            }),
-          )
-        : this.tools;
+      const tools = this.tools;
+      tools._exclude(this._onEnterIgnoredTools(tools));
 
       const task = this.createSpeechTask({
         taskFn: (abortController: AbortController) =>
@@ -2234,6 +2223,19 @@ export class AgentActivity implements RecognitionHooks {
     }
   }
 
+  _onEnterIgnoredTools(toolCtx: ToolContext): Tool[] {
+    const onEnterData = onEnterStorage.getStore();
+    if (onEnterData?.agent !== this.agent || onEnterData.session !== this.agentSession) {
+      return [];
+    }
+
+    return toolCtx
+      .flatten()
+      .filter(
+        (tool): tool is Tool => isFunctionTool(tool) && !!(tool.flags & ToolFlag.IGNORE_ON_ENTER),
+      );
+  }
+
   /**
    * Commit a user turn whose reply is being skipped: append the transcript to the
    * agent chat context (when non-empty) without triggering reply generation.
@@ -3910,11 +3912,21 @@ export class AgentActivity implements RecognitionHooks {
     }
 
     const originalToolChoice = this.toolChoice;
-    if (toolChoice !== undefined) {
-      this.realtimeSession.updateOptions({ toolChoice });
-    }
-
+    let originalTools: ToolContext | undefined;
     try {
+      const sessionTools = this.realtimeSession.tools;
+      const onEnterIgnoredTools = this._onEnterIgnoredTools(sessionTools);
+      if (onEnterIgnoredTools.length > 0) {
+        originalTools = sessionTools;
+        const turnTools = sessionTools._flattenedCopy();
+        turnTools._exclude(onEnterIgnoredTools);
+        await this.realtimeSession.updateTools(turnTools);
+      }
+
+      if (toolChoice !== undefined) {
+        this.realtimeSession.updateOptions({ toolChoice });
+      }
+
       const generateReplyAbortController = new AbortController();
       const generationPromise = this.realtimeSession.generateReply(
         instructions !== undefined
@@ -3940,7 +3952,18 @@ export class AgentActivity implements RecognitionHooks {
     } finally {
       // reset toolChoice value
       if (toolChoice !== undefined && toolChoice !== originalToolChoice) {
-        this.realtimeSession.updateOptions({ toolChoice: originalToolChoice });
+        try {
+          this.realtimeSession.updateOptions({ toolChoice: originalToolChoice });
+        } catch (error) {
+          this.logger.error({ error }, 'failed to reset tool_choice');
+        }
+      }
+      if (originalTools !== undefined) {
+        try {
+          await this.realtimeSession.updateTools(originalTools);
+        } catch (error) {
+          this.logger.error({ error }, 'failed to reset tools');
+        }
       }
     }
   }