.NET: Remove FunctionCalls and Tool Messages from Handoff passed messages (#3811)

* Fix handoff orchestration not passing user message to handoff target agent (#3161)

Filter out internal handoff function call and tool result messages before
passing conversation history to the target agent's LLM. These messages
confused the model into ignoring the original user question.

* Add handoff tool call filtering behavior and enhance workflow builder

- Introduced HandoffToolCallFilteringBehavior enum to specify filtering behavior for tool call contents in handoff workflows.
- Updated HandoffsWorkflowBuilder to support customizable handoff instructions and tool call filtering behavior.
- Enhanced HandoffAgentExecutor to utilize new filtering options for improved message handling during agent handoffs.

* Enhance handoff message filtering logic and add unit tests for filtering behaviors

* Refactor HandoffMessagesFilter to remove unused handoff function names and enhance filtering logic for non-handoff function calls

* Refactor HandoffMessagesFilter to streamline FilterCandidateState initialization and improve clarity

* Refactor HandoffMessagesFilter to improve filtering logic and add integration tests for handoff workflows

* fix: HandoffAgentExecutor tests

Author: lokitoth

dotnet/src/Microsoft.Agents.AI.Workflows/HandoffToolCallFilteringBehavior.cs

@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.AI;
+
+namespace Microsoft.Agents.AI.Workflows;
+
+/// <summary>
+/// Specifies the behavior for filtering <see cref="FunctionCallContent"/> and <see cref="ChatRole.Tool"/> contents from
+/// <see cref="ChatMessage"/>s flowing through a handoff workflow. This can be used to prevent agents from seeing external
+/// tool calls.
+/// </summary>
+public enum HandoffToolCallFilteringBehavior
+{
+    /// <summary>
+    /// Do not filter <see cref="FunctionCallContent"/> and <see cref="ChatRole.Tool"/> contents.
+    /// </summary>
+    None,
+
+    /// <summary>
+    /// Filter only handoff-related <see cref="FunctionCallContent"/> and <see cref="ChatRole.Tool"/> contents.
+    /// </summary>
+    HandoffOnly,
+
+    /// <summary>
+    /// Filter all <see cref="FunctionCallContent"/> and <see cref="ChatRole.Tool"/> contents.
+    /// </summary>
+    All
+}

dotnet/src/Microsoft.Agents.AI.Workflows/HandoffsWorkflowBuilder.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using Microsoft.Agents.AI.Workflows.Specialized;
+using Microsoft.Extensions.AI;
 using Microsoft.Shared.Diagnostics;
 
 namespace Microsoft.Agents.AI.Workflows;
@@ -16,6 +17,7 @@ public sealed class HandoffsWorkflowBuilder
     private readonly AIAgent _initialAgent;
     private readonly Dictionary<AIAgent, HashSet<HandoffTarget>> _targets = [];
     private readonly HashSet<AIAgent> _allAgents = new(AIAgentIDEqualityComparer.Instance);
+    private HandoffToolCallFilteringBehavior _toolCallFilteringBehavior = HandoffToolCallFilteringBehavior.HandoffOnly;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="HandoffsWorkflowBuilder"/> class with no handoff relationships.
@@ -34,14 +36,38 @@ internal HandoffsWorkflowBuilder(AIAgent initialAgent)
     /// By default, simple instructions are included. This may be set to <see langword="null"/> to avoid including
     /// any additional instructions, or may be customized to provide more specific guidance.
     /// </remarks>
-    public string? HandoffInstructions { get; set; } =
-         $"""
+    public string? HandoffInstructions { get; private set; } = DefaultHandoffInstructions;
+
+    private const string DefaultHandoffInstructions =
+        $"""
               You are one agent in a multi-agent system. You can hand off the conversation to another agent if appropriate. Handoffs are achieved
               by calling a handoff function, named in the form `{FunctionPrefix}<agent_id>`; the description of the function provides details on the
               target agent of that handoff. Handoffs between agents are handled seamlessly in the background; never mention or narrate these handoffs
               in your conversation with the user.
               """;
 
+    /// <summary>
+    /// Sets additional instructions to provide to an agent that has handoffs about how and when to
+    /// perform them.
+    /// </summary>
+    /// <param name="instructions">The instructions to provide, or <see langword="null"/> to restore the default instructions.</param>
+    public HandoffsWorkflowBuilder WithHandoffInstructions(string? instructions)
+    {
+        this.HandoffInstructions = instructions ?? DefaultHandoffInstructions;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets the behavior for filtering <see cref="FunctionCallContent"/> and <see cref="ChatRole.Tool"/> contents from
+    /// <see cref="ChatMessage"/>s flowing through the handoff workflow. Defaults to <see cref="HandoffToolCallFilteringBehavior.HandoffOnly"/>.
+    /// </summary>
+    /// <param name="behavior">The filtering behavior to apply.</param>
+    public HandoffsWorkflowBuilder WithToolCallFilteringBehavior(HandoffToolCallFilteringBehavior behavior)
+    {
+        this._toolCallFilteringBehavior = behavior;
+        return this;
+    }
+
     /// <summary>
     /// Adds handoff relationships from a source agent to one or more target agents.
     /// </summary>
@@ -149,8 +175,10 @@ public Workflow Build()
         HandoffsEndExecutor end = new();
         WorkflowBuilder builder = new(start);
 
+        HandoffAgentExecutorOptions options = new(this.HandoffInstructions, this._toolCallFilteringBehavior);
+
         // Create an AgentExecutor for each again.
-        Dictionary<string, HandoffAgentExecutor> executors = this._allAgents.ToDictionary(a => a.Id, a => new HandoffAgentExecutor(a, this.HandoffInstructions));
+        Dictionary<string, HandoffAgentExecutor> executors = this._allAgents.ToDictionary(a => a.Id, a => new HandoffAgentExecutor(a, options));
 
         // Connect the start executor to the initial agent.
         builder.AddEdge(start, executors[this._initialAgent.Id]);

dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/HandoffAgentExecutor.cs

@@ -12,10 +12,155 @@
 
 namespace Microsoft.Agents.AI.Workflows.Specialized;
 
+internal sealed class HandoffAgentExecutorOptions
+{
+    public HandoffAgentExecutorOptions(string? handoffInstructions, HandoffToolCallFilteringBehavior toolCallFilteringBehavior)
+    {
+        this.HandoffInstructions = handoffInstructions;
+        this.ToolCallFilteringBehavior = toolCallFilteringBehavior;
+    }
+
+    public string? HandoffInstructions { get; set; }
+
+    public HandoffToolCallFilteringBehavior ToolCallFilteringBehavior { get; set; } = HandoffToolCallFilteringBehavior.HandoffOnly;
+}
+
+internal sealed class HandoffMessagesFilter
+{
+    private readonly HandoffToolCallFilteringBehavior _filteringBehavior;
+
+    public HandoffMessagesFilter(HandoffToolCallFilteringBehavior filteringBehavior)
+    {
+        this._filteringBehavior = filteringBehavior;
+    }
+
+    internal static bool IsHandoffFunctionName(string name)
+    {
+        return name.StartsWith(HandoffsWorkflowBuilder.FunctionPrefix, StringComparison.Ordinal);
+    }
+
+    public IEnumerable<ChatMessage> FilterMessages(List<ChatMessage> messages)
+    {
+        if (this._filteringBehavior == HandoffToolCallFilteringBehavior.None)
+        {
+            return messages;
+        }
+
+        Dictionary<string, FilterCandidateState> filteringCandidates = new();
+        List<ChatMessage> filteredMessages = [];
+        HashSet<int> messagesToRemove = [];
+
+        bool filterHandoffOnly = this._filteringBehavior == HandoffToolCallFilteringBehavior.HandoffOnly;
+        foreach (ChatMessage unfilteredMessage in messages)
+        {
+            ChatMessage filteredMessage = unfilteredMessage.Clone();
+
+            // .Clone() is shallow, so we cannot modify the contents of the cloned message in place.
+            List<AIContent> contents = [];
+            contents.Capacity = unfilteredMessage.Contents?.Count ?? 0;
+            filteredMessage.Contents = contents;
+
+            // Because this runs after the role changes from assistant to user for the target agent, we cannot rely on tool calls
+            // originating only from messages with the Assistant role. Instead, we need to inspect the contents of all non-Tool (result)
+            // FunctionCallContent.
+            if (unfilteredMessage.Role != ChatRole.Tool)
+            {
+                for (int i = 0; i < unfilteredMessage.Contents!.Count; i++)
+                {
+                    AIContent content = unfilteredMessage.Contents[i];
+                    if (content is not FunctionCallContent fcc || (filterHandoffOnly && !IsHandoffFunctionName(fcc.Name)))
+                    {
+                        filteredMessage.Contents.Add(content);
+
+                        // Track non-handoff function calls so their tool results are preserved in HandoffOnly mode
+                        if (filterHandoffOnly && content is FunctionCallContent nonHandoffFcc)
+                        {
+                            filteringCandidates[nonHandoffFcc.CallId] = new FilterCandidateState(nonHandoffFcc.CallId)
+                            {
+                                IsHandoffFunction = false,
+                            };
+                        }
+                    }
+                    else if (filterHandoffOnly)
+                    {
+                        if (!filteringCandidates.TryGetValue(fcc.CallId, out FilterCandidateState? candidateState))
+                        {
+                            filteringCandidates[fcc.CallId] = new FilterCandidateState(fcc.CallId)
+                            {
+                                IsHandoffFunction = true,
+                            };
+                        }
+                        else
+                        {
+                            candidateState.IsHandoffFunction = true;
+                            (int messageIndex, int contentIndex) = candidateState.FunctionCallResultLocation!.Value;
+                            ChatMessage messageToFilter = filteredMessages[messageIndex];
+                            messageToFilter.Contents.RemoveAt(contentIndex);
+                            if (messageToFilter.Contents.Count == 0)
+                            {
+                                messagesToRemove.Add(messageIndex);
+                            }
+                        }
+                    }
+                    else
+                    {
+                        // All mode: strip all FunctionCallContent
+                    }
+                }
+            }
+            else
+            {
+                if (!filterHandoffOnly)
+                {
+                    continue;
+                }
+
+                for (int i = 0; i < unfilteredMessage.Contents!.Count; i++)
+                {
+                    AIContent content = unfilteredMessage.Contents[i];
+                    if (content is not FunctionResultContent frc
+                        || (filteringCandidates.TryGetValue(frc.CallId, out FilterCandidateState? candidateState)
+                            && candidateState.IsHandoffFunction is false))
+                    {
+                        // Either this is not a function result content, so we should let it through, or it is a FRC that
+                        // we know is not related to a handoff call. In either case, we should include it.
+                        filteredMessage.Contents.Add(content);
+                    }
+                    else if (candidateState is null)
+                    {
+                        // We haven't seen the corresponding function call yet, so add it as a candidate to be filtered later
+                        filteringCandidates[frc.CallId] = new FilterCandidateState(frc.CallId)
+                        {
+                            FunctionCallResultLocation = (filteredMessages.Count, filteredMessage.Contents.Count),
+                        };
+                    }
+                    // else we have seen the corresponding function call and it is a handoff, so we should filter it out.
+                }
+            }
+
+            if (filteredMessage.Contents.Count > 0)
+            {
+                filteredMessages.Add(filteredMessage);
+            }
+        }
+
+        return filteredMessages.Where((_, index) => !messagesToRemove.Contains(index));
+    }
+
+    private class FilterCandidateState(string callId)
+    {
+        public (int MessageIndex, int ContentIndex)? FunctionCallResultLocation { get; set; }
+
+        public string CallId => callId;
+
+        public bool? IsHandoffFunction { get; set; }
+    }
+}
+
 /// <summary>Executor used to represent an agent in a handoffs workflow, responding to <see cref="HandoffState"/> events.</summary>
 internal sealed class HandoffAgentExecutor(
     AIAgent agent,
-    string? handoffInstructions) : Executor<HandoffState, HandoffState>(agent.GetDescriptiveId(), declareCrossRunShareable: true), IResettableExecutor
+    HandoffAgentExecutorOptions options) : Executor<HandoffState, HandoffState>(agent.GetDescriptiveId(), declareCrossRunShareable: true), IResettableExecutor
 {
     private static readonly JsonElement s_handoffSchema = AIFunctionFactory.Create(
         ([Description("The reason for the handoff")] string? reasonForHandoff) => { }).JsonSchema;
@@ -39,7 +184,7 @@ public void Initialize(
                     ChatOptions = new()
                     {
                         AllowMultipleToolCalls = false,
-                        Instructions = handoffInstructions,
+                        Instructions = options.HandoffInstructions,
                         Tools = [],
                     },
                 };
@@ -69,10 +214,19 @@ public override async ValueTask<HandoffState> HandleAsync(HandoffState message,
 
         List<ChatMessage>? roleChanges = allMessages.ChangeAssistantToUserForOtherParticipants(this._agent.Name ?? this._agent.Id);
 
-        await foreach (var update in this._agent.RunStreamingAsync(allMessages,
+        // If a handoff was invoked by a previous agent, filter out the handoff function
+        // call and tool result messages before sending to the underlying agent. These
+        // are internal workflow mechanics that confuse the target model into ignoring the
+        // original user question.
+        HandoffMessagesFilter handoffMessagesFilter = new(options.ToolCallFilteringBehavior);
+        IEnumerable<ChatMessage> messagesForAgent = message.InvokedHandoff is not null
+            ? handoffMessagesFilter.FilterMessages(allMessages)
+            : allMessages;
+
+        await foreach (var update in this._agent.RunStreamingAsync(messagesForAgent,
                                                                    options: this._agentOptions,
                                                                    cancellationToken: cancellationToken)
-                                                 .ConfigureAwait(false))
+                                                .ConfigureAwait(false))
         {
             await AddUpdateAsync(update, cancellationToken).ConfigureAwait(false);