mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
.NET: Fix FoundryAgents_Step15_ComputerUse sample for Azure Agents API (#3989)
* Fix FoundryAgents_Step15_ComputerUse sample for Azure Agents API The Azure Agents API rejects previous_response_id alongside computer_call_output items, unlike the vanilla OpenAI Responses API. This fix: - Send all prior response output items (reasoning, computer_call, etc.) as input items in follow-up calls so the API has full conversation context - Create a fresh session per call to avoid ConversationId/previous_response_id - Use currentCallId instead of initialCallId for computer_call_output - Clear ContinuationToken after polling to prevent stale tokens - Remove unused initialCallId tracking variable Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address comments --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
Unverified
parent
be88da3529
commit
c3eca2567a
+28
-14
@@ -86,8 +86,6 @@ internal sealed class Program
|
||||
AllowBackgroundResponses = true,
|
||||
};
|
||||
|
||||
AgentSession session = await agent.CreateSessionAsync();
|
||||
|
||||
ChatMessage message = new(ChatRole.User, [
|
||||
new TextContent("I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete."),
|
||||
new DataContent(new BinaryData(screenshots["browser_search"]), "image/png")
|
||||
@@ -96,6 +94,11 @@ internal sealed class Program
|
||||
// Initial request with screenshot - start with Bing search page
|
||||
Console.WriteLine("Starting computer automation session (initial screenshot: cua_browser_search.png)...");
|
||||
|
||||
// IMPORTANT: Computer-use with the Azure Agents API differs from the vanilla OpenAI Responses API.
|
||||
// The Azure Agents API rejects requests that include previous_response_id alongside
|
||||
// computer_call_output items. To work around this, each call uses a fresh session (avoiding
|
||||
// previous_response_id) and re-sends the full conversation context as input items instead.
|
||||
AgentSession session = await agent.CreateSessionAsync();
|
||||
AgentResponse response = await agent.RunAsync(message, session: session, options: runOptions);
|
||||
|
||||
// Main interaction loop
|
||||
@@ -103,7 +106,6 @@ internal sealed class Program
|
||||
int iteration = 0;
|
||||
// Initialize state machine
|
||||
SearchState currentState = SearchState.Initial;
|
||||
string initialCallId = string.Empty;
|
||||
|
||||
while (true)
|
||||
{
|
||||
@@ -119,6 +121,9 @@ internal sealed class Program
|
||||
response = await agent.RunAsync(session, runOptions);
|
||||
}
|
||||
|
||||
// Clear the continuation token so the next RunAsync call is a fresh request.
|
||||
runOptions.ContinuationToken = null;
|
||||
|
||||
Console.WriteLine($"Agent response received (ID: {response.ResponseId})");
|
||||
|
||||
if (iteration >= MaxIterations)
|
||||
@@ -148,12 +153,6 @@ internal sealed class Program
|
||||
ComputerCallAction action = firstComputerCall.Action;
|
||||
string currentCallId = firstComputerCall.CallId;
|
||||
|
||||
// Set the initial computer call ID for tracking and subsequent responses.
|
||||
if (string.IsNullOrEmpty(initialCallId))
|
||||
{
|
||||
initialCallId = currentCallId;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Processing computer call (ID: {currentCallId})");
|
||||
|
||||
// Simulate executing the action and taking a screenshot
|
||||
@@ -162,16 +161,31 @@ internal sealed class Program
|
||||
|
||||
Console.WriteLine("Sending action result back to agent...");
|
||||
|
||||
AIContent content = new()
|
||||
// Build the follow-up messages with full conversation context.
|
||||
// The Azure Agents API rejects previous_response_id when computer_call_output items are
|
||||
// present, so we must re-send all prior output items (reasoning, computer_call, etc.)
|
||||
// as input items alongside the computer_call_output to maintain conversation continuity.
|
||||
List<ChatMessage> followUpMessages = [];
|
||||
|
||||
// Re-send all response output items as an assistant message so the API has full context
|
||||
List<AIContent> priorOutputContents = response.Messages
|
||||
.SelectMany(m => m.Contents)
|
||||
.ToList();
|
||||
followUpMessages.Add(new ChatMessage(ChatRole.Assistant, priorOutputContents));
|
||||
|
||||
// Add the computer_call_output as a user message
|
||||
AIContent callOutput = new()
|
||||
{
|
||||
RawRepresentation = new ComputerCallOutputResponseItem(
|
||||
initialCallId,
|
||||
currentCallId,
|
||||
output: ComputerCallOutput.CreateScreenshotOutput(new BinaryData(screenInfo.ImageBytes), "image/png"))
|
||||
};
|
||||
followUpMessages.Add(new ChatMessage(ChatRole.User, [callOutput]));
|
||||
|
||||
// Follow-up message with action result and new screenshot
|
||||
message = new(ChatRole.User, [content]);
|
||||
response = await agent.RunAsync(message, session: session, options: runOptions);
|
||||
// Create a fresh session so ConversationId does not carry over a previous_response_id.
|
||||
// Without this, the Azure Agents API returns an error when computer_call_output is present.
|
||||
session = await agent.CreateSessionAsync();
|
||||
response = await agent.RunAsync(followUpMessages, session: session, options: runOptions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+11
@@ -2,6 +2,17 @@
|
||||
|
||||
This sample demonstrates how to use the computer use tool with AI agents. The computer use tool allows agents to interact with a computer environment by viewing the screen, controlling the mouse and keyboard, and performing various actions to help complete tasks.
|
||||
|
||||
> [!NOTE]
|
||||
> **Azure Agents API vs. vanilla OpenAI Responses API behavior:**
|
||||
> The Azure Agents API rejects requests that include `previous_response_id` alongside
|
||||
> `computer_call_output` items — unlike the vanilla OpenAI Responses API, which accepts them.
|
||||
> This sample works around the limitation by creating a **fresh session for each follow-up call**
|
||||
> (so no `previous_response_id` is carried over) and re-sending all prior response output items
|
||||
> (reasoning, computer_call, etc.) as input items to preserve full conversation context.
|
||||
> Additionally, the sample uses the **current** `CallId` from each computer call response
|
||||
> (not the initial one) and clears the `ContinuationToken` after polling completes to prevent
|
||||
> stale tokens from affecting subsequent requests.
|
||||
|
||||
## What this sample demonstrates
|
||||
|
||||
- Creating agents with computer use capabilities
|
||||
|
||||
Reference in New Issue
Block a user