diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml index 2920aaa5bd..14c92694ff 100644 --- a/.github/actions/sample-validation-setup/action.yml +++ b/.github/actions/sample-validation-setup/action.yml @@ -34,7 +34,7 @@ runs: - name: Test Copilot CLI shell: bash - run: copilot -p "What can you do in one sentence?" + run: copilot --version && copilot -p "What can you do in one sentence?" - name: Azure CLI Login uses: azure/login@v2 diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 63f95a78c3..7ce2219573 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -67,11 +67,13 @@ jobs: # Azure AI configuration AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} - AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} + AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME }} # OpenAI configuration OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} @@ -97,6 +99,8 @@ jobs: - name: Create .env for samples run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env @@ -125,6 +129,7 @@ jobs: environment: integration env: OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} + OPENAI_MODEL: ${{ vars.OPENAI__CHATMODELID }} OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} defaults: @@ -144,6 +149,7 @@ jobs: - name: Create .env for samples run: | echo "OPENAI_API_KEY=$OPENAI_API_KEY" >> .env + echo "OPENAI_MODEL=$OPENAI_MODEL" >> .env echo "OPENAI_CHAT_MODEL_ID=$OPENAI_CHAT_MODEL_ID" >> .env echo "OPENAI_RESPONSES_MODEL_ID=$OPENAI_RESPONSES_MODEL_ID" >> .env @@ -158,8 +164,8 @@ jobs: name: validation-report-02-agents-openai path: python/samples/sample_validation/reports/ - validate-02-agents-azure-openai: - name: Validate 02-agents/providers/azure_openai + validate-02-agents-azure: + name: Validate 02-agents/providers/azure runs-on: ubuntu-latest environment: integration env: @@ -190,93 +196,13 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_openai --save-report --report-name 02-agents-azure-openai + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure - name: Upload validation report uses: actions/upload-artifact@v7 if: always() with: - name: validation-report-02-agents-azure-openai - path: python/samples/sample_validation/reports/ - - validate-02-agents-azure-ai: - name: Validate 02-agents/providers/azure_ai - runs-on: ubuntu-latest - environment: integration - env: - AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} - AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__CHATDEPLOYMENTNAME }} - AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} - BING_CONNECTION_ID: ${{ secrets.BING_CONNECTION_ID }} - defaults: - run: - working-directory: python - steps: - - uses: actions/checkout@v6 - - - name: Setup environment - uses: ./.github/actions/sample-validation-setup - with: - azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} - azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - os: ${{ runner.os }} - - - name: Create .env for samples - run: | - echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env - echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env - echo "AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME=$AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME" >> .env - echo "AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME=$AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME" >> .env - echo "BING_CONNECTION_ID=$BING_CONNECTION_ID" >> .env - - - name: Run sample validation - run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai --save-report --report-name 02-agents-azure-ai - - - name: Upload validation report - uses: actions/upload-artifact@v7 - if: always() - with: - name: validation-report-02-agents-azure-ai - path: python/samples/sample_validation/reports/ - - validate-02-agents-azure-ai-agent: - name: Validate 02-agents/providers/azure_ai_agent - runs-on: ubuntu-latest - environment: integration - env: - AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} - AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} - defaults: - run: - working-directory: python - steps: - - uses: actions/checkout@v6 - - - name: Setup environment - uses: ./.github/actions/sample-validation-setup - with: - azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} - azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - os: ${{ runner.os }} - - - name: Create .env for samples - run: | - echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env - echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env - - - name: Run sample validation - run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure_ai_agent --save-report --report-name 02-agents-azure-ai-agent - - - name: Upload validation report - uses: actions/upload-artifact@v7 - if: always() - with: - name: validation-report-02-agents-azure-ai-agent + name: validation-report-02-agents-azure path: python/samples/sample_validation/reports/ validate-02-agents-anthropic: @@ -409,11 +335,13 @@ jobs: name: validation-report-02-agents-ollama path: python/samples/sample_validation/reports/ - validate-02-agents-foundry-local: - name: Validate 02-agents/providers/foundry_local - if: false # Temporarily disabled - requires local Foundry setup + validate-02-agents-foundry: + name: Validate 02-agents/providers/foundry runs-on: ubuntu-latest environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -428,15 +356,20 @@ jobs: azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} os: ${{ runner.os }} + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry_local --save-report --report-name 02-agents-foundry-local + cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry - name: Upload validation report uses: actions/upload-artifact@v7 if: always() with: - name: validation-report-02-agents-foundry-local + name: validation-report-02-agents-foundry path: python/samples/sample_validation/reports/ validate-02-agents-copilotstudio: @@ -516,6 +449,8 @@ jobs: environment: integration env: # Azure AI configuration + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} AZURE_AI_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_PROJECT_ENDPOINT }} AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration @@ -538,6 +473,8 @@ jobs: - name: Create .env for samples run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env echo "AZURE_AI_PROJECT_ENDPOINT=$AZURE_AI_PROJECT_ENDPOINT" >> .env echo "AZURE_AI_MODEL_DEPLOYMENT_NAME=$AZURE_AI_MODEL_DEPLOYMENT_NAME" >> .env echo "AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> .env @@ -759,14 +696,12 @@ jobs: - validate-01-get-started - validate-02-agents - validate-02-agents-openai - - validate-02-agents-azure-openai - - validate-02-agents-azure-ai - - validate-02-agents-azure-ai-agent + - validate-02-agents-azure - validate-02-agents-anthropic - validate-02-agents-github-copilot - validate-02-agents-amazon - validate-02-agents-ollama - - validate-02-agents-foundry-local + - validate-02-agents-foundry - validate-02-agents-copilotstudio - validate-02-agents-custom - validate-03-workflows diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3c0e6dcf13..4999145e81 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,6 +74,37 @@ Contributions must maintain API signature and behavioral compatibility. Contribu that include breaking changes will be rejected. Please file an issue to discuss your idea or change if you believe that a breaking change is warranted. +#### Automated API Compatibility Validation + +The .NET projects use [Package Validation](https://learn.microsoft.com/dotnet/fundamentals/package-validation/overview) +to automatically detect API breaking changes. This validation runs during `dotnet build` +(Release configuration) and `dotnet pack`, comparing the current API surface against the +latest published NuGet baseline version. + +**What gets validated:** By default, packable RC packages (`IsReleaseCandidate=true`) and +GA packages (`IsGenerallyAvailable=true`) that have a published NuGet baseline and do not +override validation settings are automatically validated. The shared baseline version and +default validation settings are defined in `dotnet/nuget/nuget-package.props`, but +individual projects may opt out (for example by setting `EnablePackageValidation=false`). + +**If the build fails with CP errors (e.g., CP0001, CP0002):** + +1. **Unintentional breaking change** — Refactor your code to maintain backward compatibility. +2. **Intentional breaking change** (approved by maintainers) — Generate a suppression file: + ```bash + dotnet build .csproj -c Release /p:ApiCompatGenerateSuppressionFile=true + ``` + This creates or updates a `CompatibilitySuppressions.xml` in the project directory. + Include this file in your PR with justification for the breaking change. + +**After each release:** + +1. Delete all `CompatibilitySuppressions.xml` files from validated projects. +2. Update `PackageValidationBaselineVersion` in `dotnet/nuget/nuget-package.props` to the + newly published version. + +For more details, see the [Package Validation diagnostic IDs](https://learn.microsoft.com/dotnet/fundamentals/package-validation/diagnostic-ids). + ### Suggested Workflow We use and recommend the following workflow: diff --git a/docs/decisions/0020-foundry-agent-type-naming.md b/docs/decisions/0020-foundry-agent-type-naming.md new file mode 100644 index 0000000000..03d43f64ac --- /dev/null +++ b/docs/decisions/0020-foundry-agent-type-naming.md @@ -0,0 +1,125 @@ +--- +status: accepted +contact: rogerbarreto +date: 2026-03-06 +deciders: rogerbarreto, alliscode +consulted: "" +informed: "" +--- + +# Foundry agent surface stays centered on `ChatClientAgent` + +## Context + +The Microsoft Foundry integration exposes two distinct usage patterns: + +1. Direct Responses usage, where callers provide model, instructions, and tools at runtime. +2. Server-side versioned agents, where callers create and manage `AgentVersion` resources through `AIProjectClient.Agents`. + +We briefly explored adding public wrapper types such as `FoundryAgent`, `FoundryVersionedAgent`, and `FoundryResponsesChatClient` to make those paths feel more specialized. That direction created extra public types, duplicated existing `ChatClientAgent` behavior, and pushed samples toward compatibility helpers instead of the native Azure SDK flow. + +## Decision + +Keep the public surface centered on `ChatClientAgent`. + +- Direct Responses scenarios use `AIProjectClient.AsAIAgent(...)`. +- Server-side versioned scenarios use native `AIProjectClient.Agents` APIs to create or retrieve agent resources, then wrap `AgentRecord` or `AgentVersion` with `AIProjectClient.AsAIAgent(...)`. +- Compatibility helpers such as `AIProjectClient.CreateAIAgentAsync(...)` and `AIProjectClient.GetAIAgentAsync(...)` remain only as obsolete migration shims. +- Public wrapper types `FoundryAgent`, `FoundryVersionedAgent`, `FoundryResponsesChatClient`, and `FoundryResponsesChatClientAgent` are not part of the chosen direction. + +## Why + +- `ChatClientAgent` is already the framework abstraction used everywhere else. +- `AIProjectClient` is the native Azure SDK entry point for versioned agent lifecycle operations. +- A single agent abstraction avoids parallel type hierarchies for the same backend. +- Samples become clearer when they show either: + - direct Responses construction via `AIProjectClient.AsAIAgent(...)`, or + - native Foundry resource management via `AIProjectClient.Agents`. + +## Consequences + +### Direct Responses path + +Use the convenience overloads on `AIProjectClient`: + +```csharp +AIProjectClient aiProjectClient = new(new Uri(endpoint), credential); + +ChatClientAgent agent = aiProjectClient.AsAIAgent( + model: deploymentName, + instructions: "You are good at telling jokes.", + name: "JokerAgent"); +``` + +Or use composed `ChatClientAgent` + +```csharp +ProjectResponsesClient projectResponsesClient = new(new Uri(endpoint), new DefaultAzureCredential(), new AgentReference($"model:{deploymentName}")); + +ChatClientAgent agent = new( + chatClient: projectResponsesClient.AsIChatClient(), + instructions: "You are good at telling jokes.", + name: "JokerAgent"); +``` + +This path is code-first and does not create a persistent server-side agent. + +### Versioned agent path + +Use the convenience overloads on `AIProjectClient`: + +```csharp +AIProjectClient aiProjectClient = new(new Uri(endpoint), credential); + +AgentVersion version = await aiProjectClient.Agents.CreateAgentVersionAsync( + "JokerAgent", + new AgentVersionCreationOptions( + new PromptAgentDefinition(deploymentName) + { + Instructions = "You are good at telling jokes." + })); + +ChatClientAgent agent = aiProjectClient.AsAIAgent(version); +``` + +Or use composed `ChatClientAgent` + +```csharp +AIProjectClient aiProjectClient = new(new Uri(endpoint), credential); + +AgentVersion version = await aiProjectClient.Agents.CreateAgentVersionAsync( + "JokerAgent", + new AgentVersionCreationOptions( + new PromptAgentDefinition(deploymentName) + { + Instructions = "You are good at telling jokes." + })); + +ProjectResponsesClient projectResponsesClient = aiProjectClient + .GetProjectOpenAIClient() + .GetProjectResponsesClientForAgent(new AgentReference(version.Name, version.Version)); + +ChatClientAgent agent = new( + chatClient: projectResponsesClient.AsIChatClient(), + name: "JokerAgent"); +``` + +### Samples + +- `FoundryAgents/` samples show the direct Responses path with `AIProjectClient.AsAIAgent(...)`. +- `FoundryVersionedAgents/` samples should show native `AIProjectClient.Agents` create/get/delete flows plus `AsAIAgent(...)`. + +### Compatibility APIs + +Obsolete helper extensions remain only to ease migration of existing code. New samples and new guidance should not be written against them. + +## Rejected direction + +Do not introduce or preserve separate public wrapper types whose main purpose is to forward to `ChatClientAgent` while carrying Foundry-specific naming. + +That approach: + +- duplicates lifecycle concepts already present on `AIProjectClient`, +- fragments the public API, +- complicates samples and docs, +- and makes migration harder by encouraging wrapper-specific affordances. diff --git a/docs/decisions/0022-chat-history-persistence-consistency.md b/docs/decisions/0022-chat-history-persistence-consistency.md index 2c760e6614..2784934817 100644 --- a/docs/decisions/0022-chat-history-persistence-consistency.md +++ b/docs/decisions/0022-chat-history-persistence-consistency.md @@ -31,8 +31,6 @@ The persistence timing and `FunctionResultContent` trimming behaviors are interr - **Per-run persistence**: When messages are batched and persisted at the end of the full run, trailing `FunctionResultContent` trimming becomes necessary to match the service's behavior. Without trimming, the stored history contains `FunctionResultContent` that the service would never have stored. -This means the trimming feature (introduced in [PR #4792](https://github.com/microsoft/agent-framework/pull/4792)) is primarily needed as a complement to per-run persistence. The `PersistChatHistoryAtEndOfRun` setting (introduced in [PR #4762](https://github.com/microsoft/agent-framework/pull/4762)) inverts the default so that per-service-call persistence is the standard behavior, and per-run persistence is opt-in. - ## Decision Drivers - **A. Consistency**: The default behavior of `ChatHistoryProvider` should produce stored history that closely matches what the underlying AI service would store, minimizing surprise when switching between framework-managed and service-managed chat history. @@ -43,33 +41,30 @@ This means the trimming feature (introduced in [PR #4792](https://github.com/mic ## Considered Options -- Option 1: Default to per-run persistence with `FunctionResultContent` trimming (opt-in to per-service-call) -- Option 2: Default to per-service-call persistence (opt-in to per-run) +- Option 1: Per-run persistence with opt-in FRC (FunctionResultContent) trimming +- Option 2: Opt-in per-service-call persistence (via `SimulateServiceStoredChatHistory`) ## Pros and Cons of the Options -### Option 1: Default to per-run persistence with `FunctionResultContent` trimming +### Option 1: Per-run persistence with opt-in FRC trimming -Keep the current default behavior of persisting chat history only at the end of the full agent run. Add `FunctionResultContent` trimming as the default to improve consistency with service storage. Provide an opt-in setting for users who want per-service-call persistence. - -Settings: -- `PersistChatHistoryAtEndOfRun` = `true` +Keep the current default behavior of persisting chat history only at the end of the full agent run. Add `FunctionResultContent` trimming as an opt-in behavior to improve consistency with service storage. - Good, because runs are atomic — chat history is only updated when the full run succeeds, satisfying driver B. - Good, because the mental model is simple: one run = one history update, satisfying driver D. - Good, because trimming trailing `FunctionResultContent` improves consistency with service storage, partially satisfying driver A. -- Good, because users can opt in to per-service-call persistence for checkpointing/recovery scenarios, satisfying drivers C and E. - Bad, because the default persistence timing still differs from the service's behavior (per-run vs. per-service-call), only partially satisfying driver A. -- Bad, because if the process crashes mid-loop, all intermediate progress from the current run is lost, not satisfying driver C by default. +- Bad, because if the process crashes mid-loop, all intermediate progress from the current run is lost, not satisfying driver C. +- Bad, because this option alone does not provide a way for users to opt into per-service-call persistence, not satisfying driver E. -### Option 2: Default to per-service-call persistence +### Option 2: Opt-in per-service-call persistence (via `SimulateServiceStoredChatHistory`) -Change the default to persist chat history after each individual service call within the FIC loop, matching the AI service's behavior. Trailing `FunctionResultContent` trimming is unnecessary with this approach (it is naturally handled). Provide an opt-in setting for users who want per-run atomicity with trimming. +Introduce an optional SimulateServiceStoredChatHistory setting to persist chat history after each individual service call within the FIC loop, matching the AI service's behavior. Trailing `FunctionResultContent` trimming is unnecessary with this approach (it is naturally handled). Settings: -- `PersistChatHistoryAtEndOfRun` = `false` (default) +- `SimulateServiceStoredChatHistory` = `true` -- Good, because the stored history matches the service's behavior by default for both timing and content, fully satisfying driver A. +- Good, because the stored history matches the service's behavior when opting in for both timing and content, fully satisfying driver A. - Good, because intermediate progress is preserved if the process is interrupted, satisfying driver C. - Good, because no separate `FunctionResultContent` trimming logic is needed, reducing complexity. - Bad, because chat history may be left in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), not satisfying driver B. A subsequent run cannot proceed without manually providing the missing `FunctionResultContent`. @@ -78,39 +73,36 @@ Settings: ## Decision Outcome -Chosen option: **Option 2 — Default to per-service-call persistence**, because it fully satisfies the consistency driver (A), naturally handles `FunctionResultContent` trimming without additional logic, and provides better recoverability for long-running tool-calling loops. Per-run persistence remains available via the `PersistChatHistoryAtEndOfRun` setting for users who prefer atomic run semantics. +Chosen option: **Option 2: Opt-in per-service-call persistence (via `SimulateServiceStoredChatHistory`)**. The existing per-run persistence behavior is retained as-is, requiring no changes from users. Per-service-call persistence is available as an opt-in feature via the `SimulateServiceStoredChatHistory` setting. This satisfies drivers B (atomicity) and D (simplicity) for the common case, while fully satisfying driver A (consistency) for users who opt into simulated service-stored behavior. Users who need per-service-call persistence for recoverability (driver C) can enable it explicitly. ### Configuration Matrix -The behavior depends on the combination of `UseProvidedChatClientAsIs` and `PersistChatHistoryAtEndOfRun`: +The behavior depends on the combination of `UseProvidedChatClientAsIs` and `SimulateServiceStoredChatHistory`: -| `UseProvidedChatClientAsIs` | `PersistChatHistoryAtEndOfRun` | Behavior | +| `UseProvidedChatClientAsIs` | `SimulateServiceStoredChatHistory` | Behavior | |---|---|---| -| `false` (default) | `false` (default) | **Per-service-call persistence.** A `ChatHistoryPersistingChatClient` middleware is automatically injected into the chat client pipeline between `FunctionInvokingChatClient` and the leaf `IChatClient`. Messages are persisted after each service call. | -| `true` | `false` | **User responsibility.** No middleware is injected because the user has provided a custom chat client stack. The user is responsible for ensuring correct persistence behavior (e.g., by including their own persisting middleware). | -| `false` | `true` | **Per-run persistence with marking.** A `ChatHistoryPersistingChatClient` middleware is injected, but configured to *mark* messages with metadata rather than store them immediately. At the end of the run, marked messages are stored. Trailing `FunctionResultContent` is trimmed. | -| `true` | `true` | **Per-run persistence with warning.** The system checks whether the custom chat client stack includes a `ChatHistoryPersistingChatClient`. If not, a warning is emitted (particularly relevant for workflow handoff scenarios where trimming cannot be guaranteed). If no `ChatHistoryPersistingChatClient` is preset, all messages are stored at the end of the run, otherwise marked messages are stored. | +| `false` (default) | `false` (default) | **Per-run persistence.** Messages are persisted at the end of the full agent run via the `ChatHistoryProvider`. | +| `false` | `true` | **Per-service-call persistence (simulated).** A `ServiceStoredSimulatingChatClient` middleware is automatically injected into the chat client pipeline between `FunctionInvokingChatClient` and the leaf `IChatClient`. Messages are persisted after each service call. A sentinel `ConversationId` causes FIC to treat the conversation as service-managed. | +| `true` | `false` | **Per-run persistence.** No middleware is injected because the user has provided a custom chat client stack. Messages are persisted at the end of the run. | +| `true` | `true` | **User responsibility.** The system checks whether the custom chat client stack includes a `ServiceStoredSimulatingChatClient`. If not, a warning is emitted — the user is expected to have added their own per-service-call persistence mechanism. End-of-run persistence is skipped. | ### Consequences -- Good, because the stored history matches the service's behavior by default for both timing and content, fully satisfying consistency (driver A). -- Good, because intermediate progress is preserved if the process is interrupted, satisfying recoverability (driver C). -- Good, because no separate `FunctionResultContent` trimming logic is needed in the default path, reducing complexity. -- Good, because marking persisted messages with metadata enables deduplication and aids debugging. -- Good, because warnings for custom chat client configurations without the persisting middleware help prevent silent failures in workflow handoff scenarios. -- Bad, because chat history may be left in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), requiring manual recovery in rare cases. -- Bad, because the mental model is more complex for the default path: a single run may produce multiple history updates. -- Neutral, because users who prefer atomic run semantics can opt in to per-run persistence via `PersistChatHistoryAtEndOfRun = true`. +- Good, because per-run persistence is atomic by default — chat history is only updated when the full run succeeds, satisfying driver B. +- Good, because the default mental model is simple: one run = one history update, satisfying driver D. +- Good, because users who opt into `SimulateServiceStoredChatHistory` get stored history that matches the service's behavior for both timing and content, fully satisfying driver A. +- Good, because per-service-call persistence preserves intermediate progress if the process is interrupted, satisfying driver C when opted in. +- Good, because no separate `FunctionResultContent` trimming logic is needed when per-service-call persistence is active — it is naturally handled. +- Good, because conflict detection (configurable via `ThrowOnChatHistoryProviderConflict`, `WarnOnChatHistoryProviderConflict`, `ClearOnChatHistoryProviderConflict`) prevents misconfiguration when a service returns a `ConversationId` alongside a configured `ChatHistoryProvider`. +- Bad, because per-service-call persistence (when opted in) may leave chat history in an incomplete state if the run fails mid-loop (e.g., `FunctionCallContent` stored without corresponding `FunctionResultContent`), requiring manual recovery in rare cases. +- Neutral, because users who want per-service-call consistency can opt in via `SimulateServiceStoredChatHistory = true`, satisfying driver E. - Neutral, because increased write frequency from per-service-call persistence may impact performance for some storage backends; this can be mitigated with a caching decorator. ### Implementation Notes #### Conversation ID Consistency -The `ChatHistoryPersistingChatClient` middleware must also update the session's `ConversationId` consistently for both response-based and conversation-based service interactions, ensuring the session always reflects the latest service-provided identifier. +We should introduce a separate `ConversationIdPersistingChatClient`, middleware which allows us to +persist response `ConversationIds` during the FICC loop. This could be used with or without +`ServiceStoredSimulatingChatClient`. -## More Information - -- [PR #4762: Persist messages during function call loop](https://github.com/microsoft/agent-framework/pull/4762) — introduces `PersistChatHistoryAfterEachServiceCall` option and `ChatHistoryPersistingChatClient` decorator -- [PR #4792: Trim final FRC to match service storage](https://github.com/microsoft/agent-framework/pull/4792) — introduces `StoreFinalFunctionResultContent` option and `FilterFinalFunctionResultContent` logic -- [Issue #2889](https://github.com/microsoft/agent-framework/issues/2889) — original issue tracking chat history persistence during function call loops diff --git a/docs/decisions/0020-foundry-evals-integration.md b/docs/decisions/0023-foundry-evals-integration.md similarity index 99% rename from docs/decisions/0020-foundry-evals-integration.md rename to docs/decisions/0023-foundry-evals-integration.md index f5b5db4db5..ea9d2f3c69 100644 --- a/docs/decisions/0020-foundry-evals-integration.md +++ b/docs/decisions/0023-foundry-evals-integration.md @@ -462,7 +462,7 @@ class FoundryEvals: ### Azure AI: FoundryEvals Constants ```python -from agent_framework_azure_ai import FoundryEvals +from agent_framework.foundry import FoundryEvals evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY] ``` diff --git a/dotnet/Directory.Build.props b/dotnet/Directory.Build.props index 2482c43013..16b69a355f 100644 --- a/dotnet/Directory.Build.props +++ b/dotnet/Directory.Build.props @@ -17,6 +17,7 @@ false + false diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 8975fb072e..978cb22ad0 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -1,4 +1,4 @@ - + @@ -115,6 +115,7 @@ + @@ -131,6 +132,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -153,35 +182,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -328,8 +328,8 @@ - + diff --git a/dotnet/nuget/nuget-package.props b/dotnet/nuget/nuget-package.props index 7b241e9d56..f2f4b59fc0 100644 --- a/dotnet/nuget/nuget-package.props +++ b/dotnet/nuget/nuget-package.props @@ -2,17 +2,19 @@ 1.0.0 - 4 + 5 $(VersionPrefix)-rc$(RCNumber) - $(VersionPrefix)-$(VersionSuffix).260311.1 - $(VersionPrefix)-preview.260311.1 - 1.0.0-rc4 + $(VersionPrefix)-$(VersionSuffix).260330.1 + $(VersionPrefix)-preview.260330.1 + 1.0.0-rc5 Debug;Release;Publish true - 0.0.1 + 1.0.0-rc4 + + true $(NoWarn);CP0003 diff --git a/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Client/Program.cs b/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Client/Program.cs index 5d770ff3fd..0c3e75cf96 100644 --- a/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Client/Program.cs +++ b/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Client/Program.cs @@ -70,7 +70,7 @@ while ((input = Console.ReadLine()) != null && !input.Equals("exit", StringCompa if (approvalRequest.AdditionalProperties != null) { - approvalResponse.AdditionalProperties = new AdditionalPropertiesDictionary(); + approvalResponse.AdditionalProperties = []; foreach (var kvp in approvalRequest.AdditionalProperties) { approvalResponse.AdditionalProperties[kvp.Key] = kvp.Value; diff --git a/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Client/ServerFunctionApprovalClientAgent.cs b/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Client/ServerFunctionApprovalClientAgent.cs index 866bbfad31..135d87bffd 100644 --- a/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Client/ServerFunctionApprovalClientAgent.cs +++ b/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Client/ServerFunctionApprovalClientAgent.cs @@ -131,9 +131,9 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent transformedContents ??= CopyContentsUpToIndex(message.Contents, contentIndex); approvalCalls.Remove(functionResult.CallId); } - else if (transformedContents != null) + else { - transformedContents.Add(content); + transformedContents?.Add(content); } } @@ -155,10 +155,10 @@ internal sealed class ServerFunctionApprovalClientAgent : DelegatingAIAgent result ??= CopyMessagesUpToIndex(messages, messageIndex); result.Add(newMessage); } - else if (result != null) + else { // We're already copying messages, so copy this unchanged message too - result.Add(message); + result?.Add(message); } // If result is null, we haven't made any changes yet, so keep processing } diff --git a/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Server/ServerFunctionApprovalServerAgent.cs b/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Server/ServerFunctionApprovalServerAgent.cs index ff3e6ffbb1..8c1f27eea9 100644 --- a/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Server/ServerFunctionApprovalServerAgent.cs +++ b/dotnet/samples/02-agents/AGUI/Step04_HumanInLoop/Server/ServerFunctionApprovalServerAgent.cs @@ -57,16 +57,10 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent throw new InvalidOperationException("Invalid request_approval tool call"); } - var request = toolCall.Arguments.TryGetValue("request", out var reqObj) && + var request = (toolCall.Arguments.TryGetValue("request", out var reqObj) && reqObj is JsonElement argsElement && argsElement.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(ApprovalRequest))) is ApprovalRequest approvalRequest && - approvalRequest != null ? approvalRequest : null; - - if (request == null) - { - throw new InvalidOperationException("Failed to deserialize approval request from tool call"); - } - + approvalRequest != null ? approvalRequest : null) ?? throw new InvalidOperationException("Failed to deserialize approval request from tool call"); return new ToolApprovalRequestContent( requestId: request.ApprovalId, new FunctionCallContent( @@ -77,17 +71,11 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent private static ToolApprovalResponseContent ConvertToolResultToApprovalResponse(FunctionResultContent result, ToolApprovalRequestContent approval, JsonSerializerOptions jsonSerializerOptions) { - var approvalResponse = result.Result is JsonElement je ? + var approvalResponse = (result.Result is JsonElement je ? (ApprovalResponse?)je.Deserialize(jsonSerializerOptions.GetTypeInfo(typeof(ApprovalResponse))) : result.Result is string str ? (ApprovalResponse?)JsonSerializer.Deserialize(str, jsonSerializerOptions.GetTypeInfo(typeof(ApprovalResponse))) : - result.Result as ApprovalResponse; - - if (approvalResponse == null) - { - throw new InvalidOperationException("Failed to deserialize approval response from tool result"); - } - + result.Result as ApprovalResponse) ?? throw new InvalidOperationException("Failed to deserialize approval response from tool result"); return approval.CreateResponse(approvalResponse.Approved); } #pragma warning restore MEAI001 @@ -121,7 +109,7 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent // Track approval ID to original call ID mapping _ = new Dictionary(); #pragma warning disable MEAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - Dictionary trackedRequestApprovalToolCalls = new(); // Remote approvals + Dictionary trackedRequestApprovalToolCalls = []; // Remote approvals for (int messageIndex = 0; messageIndex < messages.Count; messageIndex++) { var message = messages[messageIndex]; @@ -146,7 +134,7 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent }); } else if (content is FunctionResultContent toolResult && - trackedRequestApprovalToolCalls.TryGetValue(toolResult.CallId, out var approval) == true) + trackedRequestApprovalToolCalls.TryGetValue(toolResult.CallId, out var approval)) { result ??= CopyMessagesUpToIndex(messages, messageIndex); transformedContents ??= CopyContentsUpToIndex(message.Contents, j); @@ -161,9 +149,9 @@ internal sealed class ServerFunctionApprovalAgent : DelegatingAIAgent AdditionalProperties = message.AdditionalProperties }); } - else if (result != null) + else { - result.Add(message); + result?.Add(message); } } } diff --git a/dotnet/samples/02-agents/AGUI/Step05_StateManagement/Client/StatefulAgent.cs b/dotnet/samples/02-agents/AGUI/Step05_StateManagement/Client/StatefulAgent.cs index 41c94d5686..8a8062befe 100644 --- a/dotnet/samples/02-agents/AGUI/Step05_StateManagement/Client/StatefulAgent.cs +++ b/dotnet/samples/02-agents/AGUI/Step05_StateManagement/Client/StatefulAgent.cs @@ -72,10 +72,9 @@ internal sealed class StatefulAgent : DelegatingAIAgent if (content is DataContent dataContent && dataContent.MediaType == "application/json") { // Deserialize the state - TState? newState = JsonSerializer.Deserialize( + if (JsonSerializer.Deserialize( dataContent.Data.Span, - this._jsonSerializerOptions.GetTypeInfo(typeof(TState))) as TState; - if (newState != null) + this._jsonSerializerOptions.GetTypeInfo(typeof(TState))) is TState newState) { this.State = newState; } diff --git a/dotnet/samples/02-agents/AgentProviders/Agent_With_AzureAIProject/Program.cs b/dotnet/samples/02-agents/AgentProviders/Agent_With_AzureAIProject/Program.cs index aab95d5b38..b2cd14f68a 100644 --- a/dotnet/samples/02-agents/AgentProviders/Agent_With_AzureAIProject/Program.cs +++ b/dotnet/samples/02-agents/AgentProviders/Agent_With_AzureAIProject/Program.cs @@ -6,6 +6,7 @@ using Azure.AI.Projects; using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; @@ -30,14 +31,18 @@ var createdAgentVersion = aiProjectClient.Agents.CreateAgentVersion(agentName: J // agentVersion.Name = // You can use an AIAgent with an already created server side agent version. -AIAgent existingJokerAgent = aiProjectClient.AsAIAgent(createdAgentVersion); +FoundryAgent existingJokerAgent = aiProjectClient.AsAIAgent(createdAgentVersion); // You can also create another AIAgent version by providing the same name with a different definition. -AIAgent newJokerAgent = await aiProjectClient.CreateAIAgentAsync(name: JokerName, model: deploymentName, instructions: "You are extremely hilarious at telling jokes."); +AgentVersion newJokerAgentVersion = await aiProjectClient.Agents.CreateAgentVersionAsync( + JokerName, + new AgentVersionCreationOptions(new PromptAgentDefinition(model: deploymentName) { Instructions = "You are extremely hilarious at telling jokes." })); +FoundryAgent newJokerAgent = aiProjectClient.AsAIAgent(newJokerAgentVersion); // You can also get the AIAgent latest version just providing its name. -AIAgent jokerAgentLatest = await aiProjectClient.GetAIAgentAsync(name: JokerName); -var latestAgentVersion = jokerAgentLatest.GetService()!; +AgentRecord jokerAgentRecord = await aiProjectClient.Agents.GetAgentAsync(JokerName); +FoundryAgent jokerAgentLatest = aiProjectClient.AsAIAgent(jokerAgentRecord); +AgentVersion latestAgentVersion = jokerAgentRecord.GetLatestVersion(); // The AIAgent version can be accessed via the GetService method. Console.WriteLine($"Latest agent version id: {latestAgentVersion.Id}"); diff --git a/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/Agent_Step02_CodeDefinedSkills.csproj b/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/Agent_Step02_CodeDefinedSkills.csproj new file mode 100644 index 0000000000..fd3d71fe7e --- /dev/null +++ b/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/Agent_Step02_CodeDefinedSkills.csproj @@ -0,0 +1,21 @@ + + + + Exe + net10.0 + + enable + enable + $(NoWarn);MAAI001 + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/Program.cs b/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/Program.cs new file mode 100644 index 0000000000..46c58985fb --- /dev/null +++ b/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/Program.cs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates how to define Agent Skills entirely in code using AgentInlineSkill. +// No SKILL.md files are needed — skills, resources, and scripts are all defined programmatically. +// +// Three approaches are shown using a unit-converter skill: +// 1. Static resources — inline content provided via AddResource +// 2. Dynamic resources — computed at runtime via a factory delegate +// 3. Code scripts — executable delegates the agent can invoke directly + +using System.Text.Json; +using Azure.AI.OpenAI; +using Azure.Identity; +using Microsoft.Agents.AI; +using OpenAI.Responses; + +// --- Configuration --- +string endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// --- Build the code-defined skill --- +var unitConverterSkill = new AgentInlineSkill( + name: "unit-converter", + description: "Convert between common units using a multiplication factor. Use when asked to convert miles, kilometers, pounds, or kilograms.", + instructions: """ + Use this skill when the user asks to convert between units. + + 1. Review the conversion-table resource to find the factor for the requested conversion. + 2. Check the conversion-policy resource for rounding and formatting rules. + 3. Use the convert script, passing the value and factor from the table. + """) + // 1. Static Resource: conversion tables + .AddResource( + "conversion-table", + """ + # Conversion Tables + + Formula: **result = value × factor** + + | From | To | Factor | + |-------------|-------------|----------| + | miles | kilometers | 1.60934 | + | kilometers | miles | 0.621371 | + | pounds | kilograms | 0.453592 | + | kilograms | pounds | 2.20462 | + """) + // 2. Dynamic Resource: conversion policy (computed at runtime) + .AddResource("conversion-policy", () => + { + const int Precision = 4; + return $""" + # Conversion Policy + + **Decimal places:** {Precision} + **Format:** Always show both the original and converted values with units + **Generated at:** {DateTime.UtcNow:O} + """; + }) + // 3. Code Script: convert + .AddScript("convert", (double value, double factor) => + { + double result = Math.Round(value * factor, 4); + return JsonSerializer.Serialize(new { value, factor, result }); + }); + +// --- Skills Provider --- +var skillsProvider = new AgentSkillsProvider(unitConverterSkill); + +// --- Agent Setup --- +AIAgent agent = new AzureOpenAIClient(new Uri(endpoint), new DefaultAzureCredential()) + .GetResponsesClient() + .AsAIAgent(new ChatClientAgentOptions + { + Name = "UnitConverterAgent", + ChatOptions = new() + { + Instructions = "You are a helpful assistant that can convert units.", + }, + AIContextProviders = [skillsProvider], + }, + model: deploymentName); + +// --- Example: Unit conversion --- +Console.WriteLine("Converting units with code-defined skills"); +Console.WriteLine(new string('-', 60)); + +AgentResponse response = await agent.RunAsync( + "How many kilometers is a marathon (26.2 miles)? And how many pounds is 75 kilograms?"); + +Console.WriteLine($"Agent: {response.Text}"); diff --git a/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/README.md b/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/README.md new file mode 100644 index 0000000000..5b7c8747dd --- /dev/null +++ b/dotnet/samples/02-agents/AgentSkills/Agent_Step02_CodeDefinedSkills/README.md @@ -0,0 +1,52 @@ +# Code-Defined Agent Skills Sample + +This sample demonstrates how to define **Agent Skills entirely in code** using `AgentInlineSkill`. + +## What it demonstrates + +- Creating skills programmatically with `AgentInlineSkill` — no SKILL.md files needed +- **Static resources** via `AddResource` with inline content +- **Dynamic resources** via `AddResource` with a factory delegate (computed at runtime) +- **Code scripts** via `AddScript` with a delegate handler +- Using the `AgentSkillsProvider` constructor with inline skills + +## Skills Included + +### unit-converter (code-defined) + +Converts between common units using multiplication factors. Defined entirely in C# code: + +- `conversion-table` — Static resource with factor table +- `conversion-policy` — Dynamic resource with formatting rules (generated at runtime) +- `convert` — Script that performs `value × factor` conversion + +## Running the Sample + +### Prerequisites + +- .NET 10.0 SDK +- Azure OpenAI endpoint with a deployed model + +### Setup + +```bash +export AZURE_OPENAI_ENDPOINT="https://your-endpoint.openai.azure.com/" +export AZURE_OPENAI_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +### Run + +```bash +dotnet run +``` + +### Expected Output + +``` +Converting units with code-defined skills +------------------------------------------------------------ +Agent: Here are your conversions: + +1. **26.2 miles → 42.16 km** (a marathon distance) +2. **75 kg → 165.35 lbs** +``` diff --git a/dotnet/samples/02-agents/AgentSkills/README.md b/dotnet/samples/02-agents/AgentSkills/README.md index 75b850f077..6011384997 100644 --- a/dotnet/samples/02-agents/AgentSkills/README.md +++ b/dotnet/samples/02-agents/AgentSkills/README.md @@ -1,7 +1,24 @@ # AgentSkills Samples -Samples demonstrating Agent Skills capabilities. +Samples demonstrating Agent Skills capabilities. Each sample shows a different way to define and use skills. | Sample | Description | |--------|-------------| | [Agent_Step01_FileBasedSkills](Agent_Step01_FileBasedSkills/) | Define skills as `SKILL.md` files on disk with reference documents. Uses a unit-converter skill. | +| [Agent_Step02_CodeDefinedSkills](Agent_Step02_CodeDefinedSkills/) | Define skills entirely in C# code using `AgentInlineSkill`, with static/dynamic resources and scripts. | + +## Key Concepts + +### File-Based vs Code-Defined Skills + +| Aspect | File-Based | Code-Defined | +|--------|-----------|--------------| +| Definition | `SKILL.md` files on disk | `AgentInlineSkill` instances in C# | +| Resources | All files in skill directory (filtered by extension) | `AddResource` (static value or delegate-backed) | +| Scripts | Supported via script executor delegate | `AddScript` delegates | +| Discovery | Automatic from directory path | Explicit via constructor | +| Dynamic content | No (static files only) | Yes (factory delegates) | +| Reusability | Copy skill directory | Inline or shared instances | + +For single-source scenarios, use the `AgentSkillsProvider` constructors directly. To combine multiple skill types, use the `AgentSkillsProviderBuilder`. + diff --git a/dotnet/samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step01_Running/Program.cs b/dotnet/samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step01_Running/Program.cs index 3d9c715588..04df345cd6 100644 --- a/dotnet/samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step01_Running/Program.cs +++ b/dotnet/samples/02-agents/AgentWithAnthropic/Agent_Anthropic_Step01_Running/Program.cs @@ -5,20 +5,13 @@ using Anthropic; using Anthropic.Core; using Microsoft.Agents.AI; -using Microsoft.Extensions.AI; var apiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") ?? throw new InvalidOperationException("ANTHROPIC_API_KEY is not set."); var model = Environment.GetEnvironmentVariable("ANTHROPIC_CHAT_MODEL_NAME") ?? "claude-haiku-4-5"; -AIAgent agent = new AnthropicClient(new ClientOptions { ApiKey = apiKey }) +AIAgent agent = + new AnthropicClient(new ClientOptions { ApiKey = apiKey }) .AsAIAgent(model: model, instructions: "You are good at telling jokes.", name: "Joker"); // Invoke the agent and output the text result. -var response = await agent.RunAsync("Tell me a joke about a pirate."); -Console.WriteLine(response); - -// Invoke the agent with streaming support. -await foreach (var update in agent.RunStreamingAsync("Tell me a joke about a pirate.")) -{ - Console.WriteLine(update); -} +Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate.")); diff --git a/dotnet/samples/02-agents/AgentWithMemory/AgentWithMemory_Step04_MemoryUsingFoundry/Program.cs b/dotnet/samples/02-agents/AgentWithMemory/AgentWithMemory_Step04_MemoryUsingFoundry/Program.cs index 914eda330a..d6410d3308 100644 --- a/dotnet/samples/02-agents/AgentWithMemory/AgentWithMemory_Step04_MemoryUsingFoundry/Program.cs +++ b/dotnet/samples/02-agents/AgentWithMemory/AgentWithMemory_Step04_MemoryUsingFoundry/Program.cs @@ -11,6 +11,7 @@ using System.Text.Json; using Azure.AI.Projects; using Azure.Identity; using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; using Microsoft.Agents.AI.FoundryMemory; string foundryEndpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); @@ -19,6 +20,9 @@ string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLO string embeddingModelName = Environment.GetEnvironmentVariable("AZURE_AI_EMBEDDING_DEPLOYMENT_NAME") ?? "text-embedding-ada-002"; // Create an AIProjectClient for Foundry with Azure Identity authentication. +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. DefaultAzureCredential credential = new(); AIProjectClient projectClient = new(new Uri(foundryEndpoint), credential); @@ -33,11 +37,15 @@ FoundryMemoryProvider memoryProvider = new( memoryStoreName, stateInitializer: _ => new(new FoundryMemoryProviderScope("sample-user-123"))); -AIAgent agent = await projectClient.CreateAIAgentAsync(deploymentName, - options: new ChatClientAgentOptions() +FoundryAgent agent = projectClient.AsAIAgent( + new ChatClientAgentOptions() { Name = "TravelAssistantWithFoundryMemory", - ChatOptions = new() { Instructions = "You are a friendly travel assistant. Use known memories about the user when responding, and do not invent details." }, + ChatOptions = new() + { + ModelId = deploymentName, + Instructions = "You are a friendly travel assistant. Use known memories about the user when responding, and do not invent details." + }, AIContextProviders = [memoryProvider] }); diff --git a/dotnet/samples/02-agents/AgentWithMemory/README.md b/dotnet/samples/02-agents/AgentWithMemory/README.md index 87818c77d6..aa95012f68 100644 --- a/dotnet/samples/02-agents/AgentWithMemory/README.md +++ b/dotnet/samples/02-agents/AgentWithMemory/README.md @@ -1,4 +1,4 @@ -# Agent Framework Retrieval Augmented Generation (RAG) +# Agent Framework Retrieval Augmented Generation (RAG) These samples show how to create an agent with the Agent Framework that uses Memory to remember previous conversations or facts from previous conversations. @@ -10,4 +10,4 @@ These samples show how to create an agent with the Agent Framework that uses Mem |[Memory with Azure AI Foundry](./AgentWithMemory_Step04_MemoryUsingFoundry/)|This sample demonstrates how to create and run an agent that uses Azure AI Foundry's managed memory service to extract and retrieve individual memories.| |[Bounded Chat History with Overflow](./AgentWithMemory_Step05_BoundedChatHistory/)|This sample demonstrates how to create a bounded chat history provider that overflows older messages to a vector store and recalls them as memories.| -> **See also**: [Memory Search with Foundry Agents](../FoundryAgents/FoundryAgents_Step22_MemorySearch/) - demonstrates using the built-in Memory Search tool with Azure Foundry Agents. +> **See also**: [Memory Search with Foundry Agents](../AgentsWithFoundry/Agent_Step22_MemorySearch/) - demonstrates using the built-in Memory Search tool with Azure Foundry agents. diff --git a/dotnet/samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step01_Running/Program.cs b/dotnet/samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step01_Running/Program.cs index e2bd31055a..e82f420105 100644 --- a/dotnet/samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step01_Running/Program.cs +++ b/dotnet/samples/02-agents/AgentWithOpenAI/Agent_OpenAI_Step01_Running/Program.cs @@ -4,28 +4,14 @@ using System.ClientModel; using Microsoft.Agents.AI; -using OpenAI; -using OpenAI.Chat; +using OpenAI.Responses; var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new InvalidOperationException("OPENAI_API_KEY is not set."); var model = Environment.GetEnvironmentVariable("OPENAI_CHAT_MODEL_NAME") ?? "gpt-4o-mini"; -AIAgent agent = new OpenAIClient(apiKey) - .GetChatClient(model) - .AsAIAgent(instructions: "You are good at telling jokes.", name: "Joker"); +AIAgent agent = + new ResponsesClient(new ApiKeyCredential(apiKey)) + .AsAIAgent(model: model, instructions: "You are good at telling jokes.", name: "Joker"); -UserChatMessage chatMessage = new("Tell me a joke about a pirate."); - -// Invoke the agent and output the text result. -ChatCompletion chatCompletion = await agent.RunAsync([chatMessage]); -Console.WriteLine(chatCompletion.Content.Last().Text); - -// Invoke the agent with streaming support. -AsyncCollectionResult completionUpdates = agent.RunStreamingAsync([chatMessage]); -await foreach (StreamingChatCompletionUpdate completionUpdate in completionUpdates) -{ - if (completionUpdate.ContentUpdate.Count > 0) - { - Console.WriteLine(completionUpdate.ContentUpdate[0].Text); - } -} +// Once you have the agent, you can invoke it like any other AIAgent. +Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate.")); diff --git a/dotnet/samples/02-agents/AgentWithRAG/AgentWithRAG_Step02_CustomVectorStoreRAG/Program.cs b/dotnet/samples/02-agents/AgentWithRAG/AgentWithRAG_Step02_CustomVectorStoreRAG/Program.cs index 0f65121c04..2d6e30ae14 100644 --- a/dotnet/samples/02-agents/AgentWithRAG/AgentWithRAG_Step02_CustomVectorStoreRAG/Program.cs +++ b/dotnet/samples/02-agents/AgentWithRAG/AgentWithRAG_Step02_CustomVectorStoreRAG/Program.cs @@ -16,7 +16,7 @@ using Qdrant.Client; var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set."); var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; var embeddingDeploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME") ?? "text-embedding-3-large"; -var afOverviewUrl = "https://github.com/MicrosoftDocs/semantic-kernel-docs/blob/main/agent-framework/overview/agent-framework-overview.md"; +var afOverviewUrl = "https://raw.githubusercontent.com/MicrosoftDocs/semantic-kernel-docs/refs/heads/main/agent-framework/overview/index.md"; var afMigrationUrl = "https://raw.githubusercontent.com/MicrosoftDocs/semantic-kernel-docs/refs/heads/main/agent-framework/migration-guide/from-semantic-kernel/index.md"; // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. diff --git a/dotnet/samples/02-agents/AgentWithRAG/AgentWithRAG_Step04_FoundryServiceRAG/Program.cs b/dotnet/samples/02-agents/AgentWithRAG/AgentWithRAG_Step04_FoundryServiceRAG/Program.cs index c356bccbd9..e049618a72 100644 --- a/dotnet/samples/02-agents/AgentWithRAG/AgentWithRAG_Step04_FoundryServiceRAG/Program.cs +++ b/dotnet/samples/02-agents/AgentWithRAG/AgentWithRAG_Step04_FoundryServiceRAG/Program.cs @@ -4,11 +4,13 @@ using System.ClientModel; using Azure.AI.Projects; +using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; -using Microsoft.Extensions.AI; +using Microsoft.Agents.AI.AzureAI; using OpenAI; using OpenAI.Files; +using OpenAI.Responses; using OpenAI.VectorStores; var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); @@ -37,14 +39,20 @@ ClientResult vectorStoreCreate = await vectorStoreClient.CreateVect FileIds = { uploadResult.Value.Id } }); -var fileSearchTool = new HostedFileSearchTool() { Inputs = [new HostedVectorStoreContent(vectorStoreCreate.Value.Id)] }; +// Use the native OpenAI SDK FileSearchTool directly with the vector store ID. +#pragma warning disable OPENAI001 +FileSearchTool fileSearchTool = new([vectorStoreCreate.Value.Id]); +#pragma warning restore OPENAI001 -AIAgent agent = await aiProjectClient - .CreateAIAgentAsync( - model: deploymentName, - name: "AskContoso", - instructions: "You are a helpful support specialist for Contoso Outdoors. Answer questions using the provided context and cite the source document when available.", - tools: [fileSearchTool]); +AgentVersion agentVersion = await aiProjectClient.Agents.CreateAgentVersionAsync( + "AskContoso", + new AgentVersionCreationOptions( + new PromptAgentDefinition(model: deploymentName) + { + Instructions = "You are a helpful support specialist for Contoso Outdoors. Answer questions using the provided context and cite the source document when available.", + Tools = { fileSearchTool } + })); +FoundryAgent agent = aiProjectClient.AsAIAgent(agentVersion); AgentSession session = await agent.CreateSessionAsync(); diff --git a/dotnet/samples/02-agents/Agents/Agent_Step07_AsMcpTool/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step07_AsMcpTool/Program.cs index 7bc6478968..a63063b6a5 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step07_AsMcpTool/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step07_AsMcpTool/Program.cs @@ -3,6 +3,7 @@ // This sample shows how to expose an AI agent as an MCP tool. using Azure.AI.Projects; +using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.DependencyInjection; @@ -18,11 +19,17 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYME var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()); // Create a server side agent and expose it as an AIAgent. -AIAgent agent = await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - instructions: "You are good at telling jokes, and you always start each joke with 'Aye aye, captain!'.", - name: "Joker", - description: "An agent that tells jokes."); +AgentVersion agentVersion = await aiProjectClient.Agents.CreateAgentVersionAsync( + "Joker", + new AgentVersionCreationOptions( + new PromptAgentDefinition(model: deploymentName) + { + Instructions = "You are good at telling jokes, and you always start each joke with 'Aye aye, captain!'.", + }) + { + Description = "An agent that tells jokes.", + }); +AIAgent agent = aiProjectClient.AsAIAgent(agentVersion); // Convert the agent to an AIFunction and then to an MCP tool. // The agent name and description will be used as the mcp tool name and description. diff --git a/dotnet/samples/02-agents/Agents/Agent_Step11_Middleware/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step11_Middleware/Program.cs index 18969ed66e..bab09bc886 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step11_Middleware/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step11_Middleware/Program.cs @@ -189,9 +189,9 @@ async Task PIIMiddleware(IEnumerable messages, Agent // Regex patterns for PII detection (simplified for demonstration) Regex[] piiPatterns = [ - new(@"\b\d{3}-\d{3}-\d{4}\b", RegexOptions.Compiled), // Phone number (e.g., 123-456-7890) - new(@"\b[\w\.-]+@[\w\.-]+\.\w+\b", RegexOptions.Compiled), // Email address - new(@"\b[A-Z][a-z]+\s[A-Z][a-z]+\b", RegexOptions.Compiled) // Full name (e.g., John Doe) + MyRegex(), // Phone number (e.g., 123-456-7890) + EmailRegex(), // Email address + FullNameRegex() // Full name (e.g., John Doe) ]; foreach (var pattern in piiPatterns) @@ -309,3 +309,15 @@ internal sealed class DateTimeContextProvider : MessageAIContextProvider ]); } } + +internal partial class Program +{ + [GeneratedRegex(@"\b\d{3}-\d{3}-\d{4}\b", RegexOptions.Compiled)] + private static partial Regex MyRegex(); + + [GeneratedRegex(@"\b[\w\.-]+@[\w\.-]+\.\w+\b", RegexOptions.Compiled)] + private static partial Regex EmailRegex(); + + [GeneratedRegex(@"\b[A-Z][a-z]+\s[A-Z][a-z]+\b", RegexOptions.Compiled)] + private static partial Regex FullNameRegex(); +} diff --git a/dotnet/samples/02-agents/Agents/Agent_Step15_DeepResearch/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step15_DeepResearch/Program.cs index 7a76f73455..11d3f561f4 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step15_DeepResearch/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step15_DeepResearch/Program.cs @@ -17,10 +17,10 @@ var bingConnectionId = Environment.GetEnvironmentVariable("AZURE_AI_BING_CONNECT PersistentAgentsAdministrationClientOptions persistentAgentsClientOptions = new(); persistentAgentsClientOptions.Retry.NetworkTimeout = TimeSpan.FromMinutes(20); -// Get a client to create/retrieve server side agents with. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +// Get a client to create/retrieve server side agents with. PersistentAgentsClient persistentAgentsClient = new(endpoint, new DefaultAzureCredential(), persistentAgentsClientOptions); // Define and configure the Deep Research tool. diff --git a/dotnet/samples/02-agents/Agents/Agent_Step15_DeepResearch/README.md b/dotnet/samples/02-agents/Agents/Agent_Step15_DeepResearch/README.md index dc24ba4554..1848b10826 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step15_DeepResearch/README.md +++ b/dotnet/samples/02-agents/Agents/Agent_Step15_DeepResearch/README.md @@ -23,12 +23,14 @@ Before running this sample, ensure you have: Pay special attention to the purple `Note` boxes in the Azure documentation. -**Note**: The Bing Connection ID must be from the **project**, not the resource. It has the following format: +**Note**: The Bing Grounding Connection ID must be the **full ARM resource URI** from the project, not just the connection name. It has the following format: ``` -/subscriptions//resourceGroups//providers//accounts//projects//connections/ +/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects//connections/ ``` +You can find this in the Azure AI Foundry portal under **Management > Connected resources**, or retrieve it programmatically via the connections API (`.id` property). + ## Environment Variables Set the following environment variables: @@ -37,8 +39,8 @@ Set the following environment variables: # Replace with your Azure AI Foundry project endpoint $env:AZURE_AI_PROJECT_ENDPOINT="https://your-project.services.ai.azure.com/" -# Replace with your Bing connection ID from the project -$env:AZURE_AI_BING_CONNECTION_ID="/subscriptions/.../connections/your-bing-connection" +# Replace with your Bing Grounding connection ID (full ARM resource URI) +$env:AZURE_AI_BING_CONNECTION_ID="/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects//connections/" # Optional, defaults to o3-deep-research $env:AZURE_AI_REASONING_DEPLOYMENT_NAME="o3-deep-research" diff --git a/dotnet/samples/02-agents/Agents/Agent_Step17_AdditionalAIContext/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step17_AdditionalAIContext/Program.cs index e3913c9f0e..0fd21833e1 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step17_AdditionalAIContext/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step17_AdditionalAIContext/Program.cs @@ -24,12 +24,12 @@ var deploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT Func> loadNextThreeCalendarEvents = async () => { // In a real implementation, this method would connect to a calendar service - return new string[] - { + return + [ "Doctor's appointment today at 15:00", "Team meeting today at 17:00", "Birthday party today at 20:00" - }; + ]; }; // Create an agent with an AI context provider attached that aggregates two other providers: @@ -87,7 +87,7 @@ namespace SampleApp internal sealed class TodoListAIContextProvider : AIContextProvider { private static List GetTodoItems(AgentSession? session) - => session?.StateBag.GetValue>(nameof(TodoListAIContextProvider)) ?? new List(); + => session?.StateBag.GetValue>(nameof(TodoListAIContextProvider)) ?? []; private static void SetTodoItems(AgentSession? session, List items) => session?.StateBag.SetValue(nameof(TodoListAIContextProvider), items); diff --git a/dotnet/samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/Program.cs index 07382e6417..0c74ad86b1 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/Program.cs @@ -1,15 +1,16 @@ // Copyright (c) Microsoft. All rights reserved. // This sample demonstrates how the ChatClientAgent persists chat history after each individual -// call to the AI service. +// call to the AI service, using the SimulateServiceStoredChatHistory option. // When an agent uses tools, FunctionInvokingChatClient may loop multiple times // (service call → tool execution → service call), and intermediate messages (tool calls and // results) are persisted after each service call. This allows you to inspect or recover them // even if the process is interrupted mid-loop, but may also result in chat history that is not // yet finalized (e.g., tool calls without results) being persisted, which may be undesirable in some cases. // -// To opt into end-of-run persistence instead (atomic run semantics), set -// PersistChatHistoryAtEndOfRun = true on ChatClientAgentOptions. +// To use end-of-run persistence instead (atomic run semantics), remove the +// SimulateServiceStoredChatHistory = true setting (or set it to false). End-of-run +// persistence is the default behavior. // // The sample runs two multi-turn conversations: one using non-streaming (RunAsync) and one // using streaming (RunStreamingAsync), to demonstrate correct behavior in both modes. @@ -53,7 +54,7 @@ static string GetTime([Description("The city name.")] string city) => _ => $"{city}: time data not available." }; -// Create the agent — per-service-call persistence is the default behavior. +// Create the agent — per-service-call persistence is enabled via SimulateServiceStoredChatHistory. // The in-memory ChatHistoryProvider is used by default when the service does not require service stored chat // history, so for those cases, we can inspect the chat history via session.TryGetInMemoryChatHistory(). IChatClient chatClient = string.Equals(store, "TRUE", StringComparison.OrdinalIgnoreCase) ? @@ -63,6 +64,7 @@ AIAgent agent = chatClient.AsAIAgent( new ChatClientAgentOptions { Name = "WeatherAssistant", + SimulateServiceStoredChatHistory = true, ChatOptions = new() { Instructions = "You are a helpful assistant. When asked about multiple cities, call the appropriate tool for each city.", diff --git a/dotnet/samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/README.md b/dotnet/samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/README.md index d6157586f0..461f76aaf4 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/README.md +++ b/dotnet/samples/02-agents/Agents/Agent_Step19_InFunctionLoopCheckpointing/README.md @@ -1,16 +1,19 @@ # In-Function-Loop Checkpointing -This sample demonstrates how `ChatClientAgent` persists chat history after each individual call to the AI service by default. This per-service-call persistence ensures intermediate progress is saved during the function invocation loop. +This sample demonstrates how `ChatClientAgent` can persist chat history after each individual call to the AI service using the `SimulateServiceStoredChatHistory` option. This per-service-call persistence ensures intermediate progress is saved during the function invocation loop. ## What This Sample Shows -When an agent uses tools, the `FunctionInvokingChatClient` loops multiple times (service call → tool execution → service call → …). By default, chat history is persisted after each service call via the `ChatHistoryPersistingChatClient` decorator: +When an agent uses tools, the `FunctionInvokingChatClient` loops multiple times (service call → tool execution → service call → …). By enabling `SimulateServiceStoredChatHistory = true`, chat history is persisted after each service call via the `ServiceStoredSimulatingChatClient` decorator: -- A `ChatHistoryPersistingChatClient` decorator is automatically inserted into the chat client pipeline +- A `ServiceStoredSimulatingChatClient` decorator is inserted into the chat client pipeline +- Before each service call, the decorator loads history from the `ChatHistoryProvider` and prepends it to the request - After each service call, the decorator notifies the `ChatHistoryProvider` (and any `AIContextProvider` instances) with the new messages - Only **new** messages are sent to providers on each notification — messages that were already persisted in an earlier call within the same run are deduplicated automatically -To opt into end-of-run persistence instead (atomic run semantics), set `PersistChatHistoryAtEndOfRun = true` on `ChatClientAgentOptions`. In that mode, the decorator marks messages with metadata rather than persisting them immediately, and `ChatClientAgent` persists only the marked messages at the end of the run. +By default (without `SimulateServiceStoredChatHistory`), chat history is persisted at the end of the full agent run instead. To use per-service-call persistence, set `SimulateServiceStoredChatHistory = true` on `ChatClientAgentOptions`. + +With `SimulateServiceStoredChatHistory` = true, the behavior matches that of chat history stored in the underlying AI service exactly. Per-service-call persistence is useful for: - **Crash recovery** — if the process is interrupted mid-loop, the intermediate tool calls and results are already persisted @@ -26,7 +29,7 @@ The sample asks the agent about the weather and time in three cities. The model ``` ChatClientAgent └─ FunctionInvokingChatClient (handles tool call loop) - └─ ChatHistoryPersistingChatClient (persists after each service call) + └─ ServiceStoredSimulatingChatClient (persists after each service call) └─ Leaf IChatClient (Azure OpenAI) ``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/FoundryAgents_Step02_MultiturnConversation.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/Agent_Step00_FoundryAgentLifecycle.csproj similarity index 92% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/FoundryAgents_Step02_MultiturnConversation.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/Agent_Step00_FoundryAgentLifecycle.csproj index daf7e24494..d861331d9f 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/FoundryAgents_Step02_MultiturnConversation.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/Agent_Step00_FoundryAgentLifecycle.csproj @@ -1,4 +1,4 @@ - + Exe diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/Program.cs new file mode 100644 index 0000000000..c6b2d5c764 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/Program.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to create, use, and clean up a FoundryAgent backed by a server-side +// versioned agent in Azure AI Foundry. It demonstrates the full lifecycle: +// create agent version -> wrap as FoundryAgent -> run -> delete. + +using Azure.AI.Projects; +using Azure.AI.Projects.Agents; +using Azure.Identity; +using Microsoft.Agents.AI.AzureAI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +const string JokerName = "JokerAgent"; + +// Create the AIProjectClient to manage server-side agents. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new AzureCliCredential()); + +// Create a server-side agent version using the native SDK. +AgentVersion agentVersion = await aiProjectClient.Agents.CreateAgentVersionAsync( + JokerName, + new AgentVersionCreationOptions( + new PromptAgentDefinition(model: deploymentName) + { + Instructions = "You are good at telling jokes.", + })); + +// Wrap the agent version as a FoundryAgent using the AsAIAgent extension. +FoundryAgent agent = aiProjectClient.AsAIAgent(agentVersion); + +// Once you have the agent, you can invoke it like any other AIAgent. +Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate.")); + +// Cleanup: deletes the agent and all its versions. +await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/README.md new file mode 100644 index 0000000000..738a1d2e42 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step00_FoundryAgentLifecycle/README.md @@ -0,0 +1,23 @@ +# Agent Step 00 - FoundryAgent Lifecycle + +This sample demonstrates the full lifecycle of a `FoundryAgent` backed by a server-side versioned agent in Microsoft Foundry: create → run → delete. + +## Prerequisites + +- A Microsoft Foundry project endpoint +- A model deployment name (defaults to `gpt-4o-mini`) +- Azure CLI installed and authenticated + +## Environment Variables + +| Variable | Description | Required | +| --- | --- | --- | +| `AZURE_AI_PROJECT_ENDPOINT` | Microsoft Foundry project endpoint | Yes | +| `AZURE_AI_MODEL_DEPLOYMENT_NAME` | Model deployment name | No (defaults to `gpt-4o-mini`) | + +## Running the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step00_FoundryAgentLifecycle +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/Agent_Step01_Basics.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/Agent_Step01_Basics.csproj new file mode 100644 index 0000000000..7367c1d2f8 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/Agent_Step01_Basics.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/Program.cs new file mode 100644 index 0000000000..cd89116db7 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/Program.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to create and run a basic agent with AIProjectClient.AsAIAgent(...). + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIAgent agent = + new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) + .AsAIAgent(model: deploymentName, instructions: "You are good at telling jokes.", name: "JokerAgent"); + +// Once you have the agent, you can invoke it like any other AIAgent. +Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate.")); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/README.md new file mode 100644 index 0000000000..88eebb2a82 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step01_Basics/README.md @@ -0,0 +1,55 @@ +# Creating and Running a Basic Agent with the Responses API + +This sample demonstrates how to create and run a basic AI agent using the `ChatClientAgent`, which uses the Microsoft Foundry Responses API directly without creating server-side agent definitions. + +## What this sample demonstrates + +- Creating a `ChatClientAgent` with instructions and a model +- Running a simple single-turn conversation +- No server-side agent creation or cleanup required + +## Prerequisites + +Before you begin, ensure you have the following prerequisites: + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (for Azure credential authentication) + +**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Microsoft Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +Navigate to the AgentsWithFoundry sample directory and run: + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step01_Basics +``` + +## Alternative: Composable approach + +You can also create the same agent by composing the underlying `IChatClient` directly. This gives you full control over the chat client pipeline: + +```csharp +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = new ChatClientAgent( + chatClient: aiProjectClient.GetProjectOpenAIClient().GetProjectResponsesClient().AsIChatClient(deploymentName), + instructions: "You are good at telling jokes.", + name: "JokerAgent"); +``` + +This approach is useful when you need to customize the chat client pipeline or swap providers (e.g., Anthropic, OpenAI) while keeping the same agent code. diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/FoundryAgents_Evaluations_Step01_RedTeaming.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/Agent_Step02.1_MultiturnConversation.csproj similarity index 70% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/FoundryAgents_Evaluations_Step01_RedTeaming.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/Agent_Step02.1_MultiturnConversation.csproj index d77c0bb0d3..5e73fd236a 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/FoundryAgents_Evaluations_Step01_RedTeaming.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/Agent_Step02.1_MultiturnConversation.csproj @@ -9,8 +9,7 @@ - - + diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/Program.cs new file mode 100644 index 0000000000..15b880102f --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/Program.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to create a multi-turn conversation agent using sessions. +// Context is preserved across multiple runs via response ID chaining in the session. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) + .AsAIAgent(deploymentName, instructions: "You are good at telling jokes.", name: "JokerAgent"); + +// Create a session to maintain context across multiple runs. +AgentSession session = await agent.CreateSessionAsync(); + +// First turn +Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate.", session)); + +// Second turn — the agent remembers the first turn via the session. +Console.WriteLine(await agent.RunAsync("Now add some emojis to the joke and tell it in the voice of a pirate's parrot.", session)); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/README.md new file mode 100644 index 0000000000..a895981952 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.1_MultiturnConversation/README.md @@ -0,0 +1,36 @@ +# Multi-turn Conversation + +This sample demonstrates how to implement multi-turn conversations where context is preserved across multiple agent runs using sessions and response ID chaining. + +## What this sample demonstrates + +- Creating an agent with instructions +- Using sessions to maintain conversation context across multiple runs +- Response ID chaining for multi-turn conversations +- No server-side conversation creation required + +## Prerequisites + +Before you begin, ensure you have the following prerequisites: + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (for Azure credential authentication) + +**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Microsoft Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +Navigate to the AgentsWithFoundry sample directory and run: + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step02.1_MultiturnConversation +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/Agent_Step02.2_MultiturnWithServerConversations.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/Agent_Step02.2_MultiturnWithServerConversations.csproj new file mode 100644 index 0000000000..7367c1d2f8 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/Agent_Step02.2_MultiturnWithServerConversations.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/Program.cs new file mode 100644 index 0000000000..7a66555c18 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/Program.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use server-side conversations with a FoundryAgent. +// Server-side conversations persist on the Foundry service and are visible in the Foundry Project UI. +// Use this when you need conversation history to be stored and accessible server-side. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +FoundryAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) + .AsAIAgent(deploymentName, instructions: "You are good at telling jokes.", name: "JokerAgent"); + +// CreateConversationSessionAsync creates a server-side ProjectConversation +// that persists on the Foundry service and is visible in the Foundry Project UI. +AgentSession session = await agent.CreateConversationSessionAsync(); + +Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate.", session)); +Console.WriteLine(await agent.RunAsync("Now add some emojis to the joke and tell it in the voice of a pirate's parrot.", session)); + +// Streaming with server-side conversation context. +await foreach (AgentResponseUpdate update in agent.RunStreamingAsync("Tell me another joke, but about a ninja this time.", session)) +{ + Console.Write(update); +} + +Console.WriteLine(); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/README.md new file mode 100644 index 0000000000..9d35133b39 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step02.2_MultiturnWithServerConversations/README.md @@ -0,0 +1,36 @@ +# Multi-turn Conversation with Server-Side Conversations + +This sample demonstrates how to use server-side conversations with a `FoundryAgent`. Server-side conversations persist on the Foundry service and are visible in the Foundry Project UI, making them ideal when you need conversation history to be stored and accessible server-side. + +## What this sample demonstrates + +- Creating a `FoundryAgent` with instructions +- Using `CreateConversationSessionAsync` to create a server-side `ProjectConversation` +- Multi-turn conversations with both text and streaming output +- Server-side conversation persistence visible in the Foundry Project UI + +## Prerequisites + +Before you begin, ensure you have the following prerequisites: + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (for Azure credential authentication) + +**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Microsoft Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +Navigate to the AgentsWithFoundry sample directory and run: + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step02.2_MultiturnWithServerConversations +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/Agent_Step03_UsingFunctionTools.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/Agent_Step03_UsingFunctionTools.csproj new file mode 100644 index 0000000000..7367c1d2f8 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/Agent_Step03_UsingFunctionTools.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/Program.cs new file mode 100644 index 0000000000..7935835a24 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/Program.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample demonstrates how to use function tools. + +using System.ComponentModel; +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +[Description("Get the weather for a given location.")] +static string GetWeather([Description("The location to get the weather for.")] string location) + => $"The weather in {location} is cloudy with a high of 15°C."; + +// Define the function tool. +AITool tool = AIFunctionFactory.Create(GetWeather); + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// Create a AIAgent with function tools. +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are a helpful assistant that can get weather information.", + name: "WeatherAssistant", + tools: [tool]); + +// Non-streaming agent interaction with function tools. +AgentSession session = await agent.CreateSessionAsync(); +Console.WriteLine(await agent.RunAsync("What is the weather like in Amsterdam?", session)); + +// Streaming agent interaction with function tools. +session = await agent.CreateSessionAsync(); +await foreach (AgentResponseUpdate update in agent.RunStreamingAsync("What is the weather like in Amsterdam?", session)) +{ + Console.Write(update); +} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/README.md new file mode 100644 index 0000000000..9aa2a4ee69 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step03_UsingFunctionTools/README.md @@ -0,0 +1,37 @@ +# Using Function Tools with the Responses API + +This sample demonstrates how to use function tools with the `ChatClientAgent`, allowing the agent to call custom functions to retrieve information. + +## What this sample demonstrates + +- Creating function tools using `AIFunctionFactory` +- Passing function tools to a `ChatClientAgent` +- Running agents with function tools (text output) +- Running agents with function tools (streaming output) +- No server-side agent creation or cleanup required + +## Prerequisites + +Before you begin, ensure you have the following prerequisites: + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (for Azure credential authentication) + +**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Microsoft Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +Navigate to the AgentsWithFoundry sample directory and run: + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step03_UsingFunctionTools +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/Agent_Step04_UsingFunctionToolsWithApprovals.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/Agent_Step04_UsingFunctionToolsWithApprovals.csproj new file mode 100644 index 0000000000..7367c1d2f8 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/Agent_Step04_UsingFunctionToolsWithApprovals.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/Program.cs similarity index 69% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/Program.cs index 08051a500e..9a85dba83b 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/Program.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // This sample demonstrates how to use an agent with function tools that require a human in the loop for approvals. -// It shows both non-streaming and streaming agent interactions using weather-related tools. -// If the agent is hosted in a service, with a remote user, combine this sample with the Persisted Conversations sample to persist the chat history -// while the agent is waiting for user input. using System.ComponentModel; using Azure.AI.Projects; @@ -11,18 +8,13 @@ using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -// Create a sample function tool that the agent can use. [Description("Get the weather for a given location.")] static string GetWeather([Description("The location to get the weather for.")] string location) => $"The weather in {location} is cloudy with a high of 15°C."; -const string AssistantInstructions = "You are a helpful assistant that can get weather information."; -const string AssistantName = "WeatherAssistant"; +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. @@ -30,16 +22,16 @@ AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredent ApprovalRequiredAIFunction approvalTool = new(AIFunctionFactory.Create(GetWeather, name: nameof(GetWeather))); -// Create AIAgent directly -AIAgent agent = await aiProjectClient.CreateAIAgentAsync(name: AssistantName, model: deploymentName, instructions: AssistantInstructions, tools: [approvalTool]); +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are a helpful assistant that can get weather information.", + name: "WeatherAssistant", + tools: [approvalTool]); // Call the agent with approval-required function tools. -// The agent will request approval before invoking the function. AgentSession session = await agent.CreateSessionAsync(); AgentResponse response = await agent.RunAsync("What is the weather like in Amsterdam?", session); // Check if there are any approval requests. -// For simplicity, we are assuming here that only function approvals are pending. List approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType().ToList(); while (approvalRequests.Count > 0) @@ -53,13 +45,8 @@ while (approvalRequests.Count > 0) return new ChatMessage(ChatRole.User, [functionApprovalRequest.CreateResponse(approved)]); }); - // Pass the user input responses back to the agent for further processing. response = await agent.RunAsync(userInputMessages, session); - approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType().ToList(); } Console.WriteLine($"\nAgent: {response}"); - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/README.md new file mode 100644 index 0000000000..430c57548d --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step04_UsingFunctionToolsWithApprovals/README.md @@ -0,0 +1,30 @@ +# Using Function Tools with Approvals via the Responses API + +This sample demonstrates how to use function tools that require human-in-the-loop approval before execution. + +## What this sample demonstrates + +- Creating function tools that require approval using `ApprovalRequiredAIFunction` +- Handling approval requests from the agent +- Passing approval responses back to the agent +- No server-side agent creation or cleanup required + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step04_UsingFunctionToolsWithApprovals +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/Agent_Step05_StructuredOutput.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/Agent_Step05_StructuredOutput.csproj new file mode 100644 index 0000000000..7367c1d2f8 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/Agent_Step05_StructuredOutput.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/Program.cs similarity index 53% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/Program.cs index 3c02a4cec2..28cd4cf6bc 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/Program.cs @@ -15,29 +15,23 @@ using SampleApp; string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; -const string AssistantInstructions = "You are a helpful assistant that extracts structured information about people."; -const string AssistantName = "StructuredOutputAssistant"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -// Create ChatClientAgent directly -ChatClientAgent agent = await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - new ChatClientAgentOptions() +AIAgent agent = aiProjectClient.AsAIAgent(new ChatClientAgentOptions +{ + Name = "StructuredOutputAssistant", + ChatOptions = new() { - Name = AssistantName, - ChatOptions = new() - { - Instructions = AssistantInstructions, - ResponseFormat = Microsoft.Extensions.AI.ChatResponseFormat.ForJsonSchema() - } - }); + ModelId = deploymentName, + Instructions = "You are a helpful assistant that extracts structured information about people.", + ResponseFormat = Microsoft.Extensions.AI.ChatResponseFormat.ForJsonSchema() + } +}); -// Set PersonInfo as the type parameter of RunAsync method to specify the expected structured output from the agent and invoke the agent with some unstructured input. +// Set PersonInfo as the type parameter of RunAsync method to specify the expected structured output. AgentResponse response = await agent.RunAsync("Please provide information about John Smith, who is a 35-year-old software engineer."); // Access the structured output via the Result property of the agent response. @@ -46,39 +40,21 @@ Console.WriteLine($"Name: {response.Result.Name}"); Console.WriteLine($"Age: {response.Result.Age}"); Console.WriteLine($"Occupation: {response.Result.Occupation}"); -// Create the ChatClientAgent with the specified name, instructions, and expected structured output the agent should produce. -ChatClientAgent agentWithPersonInfo = await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - new ChatClientAgentOptions() - { - Name = AssistantName, - ChatOptions = new() - { - Instructions = AssistantInstructions, - ResponseFormat = Microsoft.Extensions.AI.ChatResponseFormat.ForJsonSchema() - } - }); +// Invoke the agent with streaming support, then deserialize the assembled response. +IAsyncEnumerable updates = agent.RunStreamingAsync("Please provide information about Jane Doe, who is a 28-year-old data scientist."); -// Invoke the agent with some unstructured input while streaming, to extract the structured information from. -IAsyncEnumerable updates = agentWithPersonInfo.RunStreamingAsync("Please provide information about John Smith, who is a 35-year-old software engineer."); - -// Assemble all the parts of the streamed output, since we can only deserialize once we have the full json, -// then deserialize the response into the PersonInfo class. PersonInfo personInfo = JsonSerializer.Deserialize((await updates.ToAgentResponseAsync()).Text, JsonSerializerOptions.Web) ?? throw new InvalidOperationException("Failed to deserialize the streamed response into PersonInfo."); -Console.WriteLine("Assistant Output:"); +Console.WriteLine("\nStreaming Assistant Output:"); Console.WriteLine($"Name: {personInfo.Name}"); Console.WriteLine($"Age: {personInfo.Age}"); Console.WriteLine($"Occupation: {personInfo.Occupation}"); -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); - namespace SampleApp { /// - /// Represents information about a person, including their name, age, and occupation, matched to the JSON schema used in the agent. + /// Represents information about a person. /// [Description("Information about a person including their name, age, and occupation")] public class PersonInfo diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/README.md new file mode 100644 index 0000000000..f5421b3f64 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step05_StructuredOutput/README.md @@ -0,0 +1,29 @@ +# Structured Output with the Responses API + +This sample demonstrates how to configure an agent to produce structured output using JSON schema. + +## What this sample demonstrates + +- Using `RunAsync()` to get typed structured output from the agent +- Deserializing streamed responses into structured types +- No server-side agent creation or cleanup required + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step05_StructuredOutput +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/Agent_Step06_PersistedConversations.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/Agent_Step06_PersistedConversations.csproj new file mode 100644 index 0000000000..7367c1d2f8 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/Agent_Step06_PersistedConversations.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/Program.cs similarity index 78% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/Program.cs index d8a5a7cd35..c9774cb1bf 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/Program.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to create and use a simple AI agent with a conversation that can be persisted to disk. +// This sample shows how to persist and resume conversations. using System.Text.Json; using Azure.AI.Projects; @@ -10,16 +10,14 @@ using Microsoft.Agents.AI; string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; -const string JokerInstructions = "You are good at telling jokes."; -const string JokerName = "JokerAgent"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -AIAgent agent = await aiProjectClient.CreateAIAgentAsync(name: JokerName, model: deploymentName, instructions: JokerInstructions); +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are good at telling jokes.", + name: "JokerAgent"); // Start a new session for the agent conversation. AgentSession session = await agent.CreateSessionAsync(); @@ -42,6 +40,3 @@ AgentSession resumedSession = await agent.DeserializeSessionAsync(reloadedSerial // Run the agent again with the resumed session. Console.WriteLine(await agent.RunAsync("Now tell the same joke in the voice of a pirate, and add some emojis to the joke.", resumedSession)); - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/README.md new file mode 100644 index 0000000000..a8cfda07ca --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step06_PersistedConversations/README.md @@ -0,0 +1,30 @@ +# Persisted Conversations with the Responses API + +This sample demonstrates how to persist and resume agent conversations using session serialization. + +## What this sample demonstrates + +- Serializing agent sessions to JSON for persistence +- Saving and loading sessions from disk +- Resuming conversations with preserved context +- No server-side agent creation or cleanup required + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step06_PersistedConversations +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/FoundryAgents_Step07_Observability.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/Agent_Step07_Observability.csproj similarity index 85% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/FoundryAgents_Step07_Observability.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/Agent_Step07_Observability.csproj index 5ceeabb204..1189939bc0 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/FoundryAgents_Step07_Observability.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/Agent_Step07_Observability.csproj @@ -9,8 +9,6 @@ - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/Program.cs similarity index 71% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/Program.cs index 257e24859f..68bfb91af0 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/Program.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to create and use a simple AI agent with Azure Foundry Agents as the backend that logs telemetry using OpenTelemetry. +// This sample shows how to add OpenTelemetry observability to an agent. using Azure.AI.Projects; using Azure.Identity; @@ -9,15 +9,11 @@ using Microsoft.Agents.AI; using OpenTelemetry; using OpenTelemetry.Trace; +string? applicationInsightsConnectionString = Environment.GetEnvironmentVariable("APPLICATIONINSIGHTS_CONNECTION_STRING"); string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; -string? applicationInsightsConnectionString = Environment.GetEnvironmentVariable("APPLICATIONINSIGHTS_CONNECTION_STRING"); -const string JokerInstructions = "You are good at telling jokes."; -const string JokerName = "JokerAgent"; - -// Create TracerProvider with console exporter -// This will output the telemetry data to the console. +// Create TracerProvider with console exporter. string sourceName = Guid.NewGuid().ToString("N"); TracerProviderBuilder tracerProviderBuilder = Sdk.CreateTracerProviderBuilder() .AddSource(sourceName) @@ -28,14 +24,16 @@ if (!string.IsNullOrWhiteSpace(applicationInsightsConnectionString)) } using var tracerProvider = tracerProviderBuilder.Build(); -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -// Define the agent you want to create. (Prompt Agent in this case) -AIAgent agent = (await aiProjectClient.CreateAIAgentAsync(name: JokerName, model: deploymentName, instructions: JokerInstructions)) +AIAgent agent = aiProjectClient + .AsAIAgent( + deploymentName, + instructions: "You are good at telling jokes.", + name: "JokerAgent") .AsBuilder() .UseOpenTelemetry(sourceName: sourceName) .Build(); @@ -48,8 +46,7 @@ Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate.", session session = await agent.CreateSessionAsync(); await foreach (AgentResponseUpdate update in agent.RunStreamingAsync("Tell me a joke about a pirate.", session)) { - Console.WriteLine(update); + Console.Write(update); } -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); +Console.WriteLine(); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/README.md new file mode 100644 index 0000000000..cb3bb729ff --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step07_Observability/README.md @@ -0,0 +1,31 @@ +# Observability with the Responses API + +This sample demonstrates how to add OpenTelemetry observability to an agent using console and Azure Monitor exporters. + +## What this sample demonstrates + +- Configuring OpenTelemetry tracing with console exporter +- Optional Azure Application Insights integration +- Using `.AsBuilder().UseOpenTelemetry()` to add telemetry to the agent +- No server-side agent creation or cleanup required + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +$env:APPLICATIONINSIGHTS_CONNECTION_STRING="..." # Optional +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step07_Observability +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/FoundryAgents_Step08_DependencyInjection.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/Agent_Step08_DependencyInjection.csproj similarity index 83% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/FoundryAgents_Step08_DependencyInjection.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/Agent_Step08_DependencyInjection.csproj index f1812befeb..72af634725 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/FoundryAgents_Step08_DependencyInjection.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/Agent_Step08_DependencyInjection.csproj @@ -11,8 +11,6 @@ - - diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/Program.cs new file mode 100644 index 0000000000..52a7d73132 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/Program.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use dependency injection to register a AIAgent and use it from a hosted service. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using SampleApp; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are good at telling jokes.", + name: "JokerAgent"); + +// Create a host builder that we will register services with and then run. +HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); + +// Add the AI agent to the service collection. +builder.Services.AddSingleton(agent); + +// Add a sample service that will use the agent to respond to user input. +builder.Services.AddHostedService(); + +// Build and run the host. +using IHost host = builder.Build(); +await host.RunAsync().ConfigureAwait(false); + +namespace SampleApp +{ + /// + /// A sample service that uses an AI agent to respond to user input. + /// + internal sealed class SampleService(AIAgent agent, IHostApplicationLifetime appLifetime) : IHostedService + { + private AgentSession? _session; + + public async Task StartAsync(CancellationToken cancellationToken) + { + this._session = await agent.CreateSessionAsync(cancellationToken); + _ = this.RunAsync(appLifetime.ApplicationStopping); + } + + public async Task RunAsync(CancellationToken cancellationToken) + { + await Task.Delay(100, cancellationToken); + + while (!cancellationToken.IsCancellationRequested) + { + Console.WriteLine("\nAgent: Ask me to tell you a joke about a specific topic. To exit just press Ctrl+C or enter without any input.\n"); + Console.Write("> "); + string? input = Console.ReadLine(); + + if (string.IsNullOrWhiteSpace(input)) + { + appLifetime.StopApplication(); + break; + } + + await foreach (AgentResponseUpdate update in agent.RunStreamingAsync(input, this._session, cancellationToken: cancellationToken)) + { + Console.Write(update); + } + + Console.WriteLine(); + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + Console.WriteLine("\nShutting down..."); + return Task.CompletedTask; + } + } +} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/README.md new file mode 100644 index 0000000000..c9ad936fd6 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step08_DependencyInjection/README.md @@ -0,0 +1,30 @@ +# Dependency Injection with the Responses API + +This sample demonstrates how to register a `ChatClientAgent` in a dependency injection container and use it from a hosted service. + +## What this sample demonstrates + +- Registering `ChatClientAgent` as an `AIAgent` in the service collection +- Using the agent from a `IHostedService` with an interactive chat loop +- Streaming responses in a hosted service context +- No server-side agent creation or cleanup required + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step08_DependencyInjection +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/FoundryAgents_Step23_LocalMCP.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/Agent_Step09_UsingMcpClientAsTools.csproj similarity index 84% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/FoundryAgents_Step23_LocalMCP.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/Agent_Step09_UsingMcpClientAsTools.csproj index 1e3e6f57e3..96cdf948fe 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/FoundryAgents_Step23_LocalMCP.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/Agent_Step09_UsingMcpClientAsTools.csproj @@ -6,16 +6,15 @@ enable enable - $(NoWarn);CA1812 - + diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/Program.cs new file mode 100644 index 0000000000..87f80297af --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/Program.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use MCP client tools with an agent. +// It connects to the Microsoft Learn MCP server via HTTP and uses its tools. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using ModelContextProtocol.Client; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// Connect to the Microsoft Learn MCP server via HTTP (Streamable HTTP transport). +Console.WriteLine("Connecting to MCP server at https://learn.microsoft.com/api/mcp ..."); + +await using McpClient mcpClient = await McpClient.CreateAsync(new HttpClientTransport(new() +{ + Endpoint = new Uri("https://learn.microsoft.com/api/mcp"), + Name = "Microsoft Learn MCP", +})); + +// Retrieve the list of tools available on the MCP server. +IList mcpTools = await mcpClient.ListToolsAsync(); +Console.WriteLine($"MCP tools available: {string.Join(", ", mcpTools.Select(t => t.Name))}"); + +List agentTools = [.. mcpTools.Cast()]; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are a helpful assistant that can help with Microsoft documentation questions. Use the Microsoft Learn MCP tool to search for documentation.", + name: "DocsAgent", + tools: agentTools); + +Console.WriteLine($"Agent '{agent.Name}' created. Asking a question...\n"); + +const string Prompt = "How does one create an Azure storage account using az cli?"; +Console.WriteLine($"User: {Prompt}\n"); +Console.WriteLine($"Agent: {await agent.RunAsync(Prompt)}"); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/README.md new file mode 100644 index 0000000000..72437e0802 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step09_UsingMcpClientAsTools/README.md @@ -0,0 +1,29 @@ +# Using MCP Client as Tools with the Responses API + +This sample shows how to use MCP (Model Context Protocol) client tools with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Connecting to an MCP server via HTTP client transport +- Retrieving MCP tools and passing them to a `ChatClientAgent` +- Using MCP tools for agent interactions without server-side agent creation + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) +- Node.js installed (for npx/MCP server) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/Agent_Step10_UsingImages.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/Agent_Step10_UsingImages.csproj new file mode 100644 index 0000000000..6064cf9334 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/Agent_Step10_UsingImages.csproj @@ -0,0 +1,21 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + PreserveNewest + + + + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/Program.cs similarity index 61% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/Program.cs index d810c8046a..2fdc150be9 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/Program.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to use Image Multi-Modality with an AI agent. +// This sample shows how to use image multi-modality with an agent. using Azure.AI.Projects; using Azure.Identity; @@ -10,29 +10,25 @@ using Microsoft.Extensions.AI; string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o"; -const string VisionInstructions = "You are a helpful agent that can analyze images"; -const string VisionName = "VisionAgent"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -// Define the agent you want to create. (Prompt Agent in this case) -AIAgent agent = await aiProjectClient.CreateAIAgentAsync(name: VisionName, model: deploymentName, instructions: VisionInstructions); +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are a helpful agent that can analyze images.", + name: "VisionAgent"); ChatMessage message = new(ChatRole.User, [ new TextContent("What do you see in this image?"), - await DataContent.LoadFromAsync("Assets/walkway.jpg"), + await DataContent.LoadFromAsync("assets/walkway.jpg"), ]); AgentSession session = await agent.CreateSessionAsync(); await foreach (AgentResponseUpdate update in agent.RunStreamingAsync(message, session)) { - Console.WriteLine(update); + Console.Write(update); } -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); +Console.WriteLine(); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/README.md new file mode 100644 index 0000000000..370bf896cf --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/README.md @@ -0,0 +1,30 @@ +# Using Images with the Responses API + +This sample demonstrates how to use image multi-modality with an agent. + +## What this sample demonstrates + +- Loading images using `DataContent.LoadFromAsync` +- Sending images alongside text to the agent +- Streaming the agent's image analysis response +- No server-side agent creation or cleanup required + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and a vision-capable model deployment (e.g., `gpt-4o`) +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step10_UsingImages +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/Assets/walkway.jpg b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/assets/walkway.jpg similarity index 100% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/Assets/walkway.jpg rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step10_UsingImages/assets/walkway.jpg diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/Agent_Step11_AsFunctionTool.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/Agent_Step11_AsFunctionTool.csproj new file mode 100644 index 0000000000..7367c1d2f8 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/Agent_Step11_AsFunctionTool.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/Program.cs similarity index 57% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/Program.cs index 585725322e..3715ab194f 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/Program.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to create and use an Azure Foundry Agents AI agent as a function tool. +// This sample shows how to use one agent as a function tool for another agent. using System.ComponentModel; using Azure.AI.Projects; @@ -8,43 +8,29 @@ using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -const string WeatherInstructions = "You answer questions about the weather."; -const string WeatherName = "WeatherAgent"; -const string MainInstructions = "You are a helpful assistant who responds in French."; -const string MainName = "MainAgent"; - [Description("Get the weather for a given location.")] static string GetWeather([Description("The location to get the weather for.")] string location) => $"The weather in {location} is cloudy with a high of 15°C."; -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -// Create the weather agent with function tools. AITool weatherTool = AIFunctionFactory.Create(GetWeather); -AIAgent weatherAgent = await aiProjectClient.CreateAIAgentAsync( - name: WeatherName, - model: deploymentName, - instructions: WeatherInstructions, +AIAgent weatherAgent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You answer questions about the weather.", + name: "WeatherAgent", tools: [weatherTool]); -// Create the main agent, and provide the weather agent as a function tool. -AIAgent agent = await aiProjectClient.CreateAIAgentAsync( - name: MainName, - model: deploymentName, - instructions: MainInstructions, +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are a helpful assistant who responds in French.", + name: "MainAgent", tools: [weatherAgent.AsAIFunction()]); // Invoke the agent and output the text result. AgentSession session = await agent.CreateSessionAsync(); Console.WriteLine(await agent.RunAsync("What is the weather like in Amsterdam?", session)); - -// Cleanup by agent name removes the agent versions created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); -await aiProjectClient.Agents.DeleteAgentAsync(weatherAgent.Name); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/README.md new file mode 100644 index 0000000000..7d361305b9 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step11_AsFunctionTool/README.md @@ -0,0 +1,30 @@ +# Agent as a Function Tool with the Responses API + +This sample demonstrates how to use one agent as a function tool for another agent. + +## What this sample demonstrates + +- Creating a specialized agent (weather) with function tools +- Exposing an agent as a function tool using `.AsAIFunction()` +- Composing agents where one agent delegates to another +- No server-side agent creation or cleanup required + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step11_AsFunctionTool +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/FoundryAgents_Step12_Middleware.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/Agent_Step12_Middleware.csproj similarity index 81% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/FoundryAgents_Step12_Middleware.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/Agent_Step12_Middleware.csproj index 9f29a8d7e6..b30baccd54 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/FoundryAgents_Step12_Middleware.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/Agent_Step12_Middleware.csproj @@ -3,15 +3,13 @@ Exe net10.0 - + enable enable - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/Program.cs similarity index 70% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/Program.cs index 824e1507b3..e37bf89639 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/Program.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows multiple middleware layers working together with Azure Foundry Agents: +// This sample shows multiple middleware layers working together with a ChatClientAgent: // agent run (PII filtering and guardrails), // function invocation (logging and result overrides), and human-in-the-loop // approval workflows for sensitive function calls. @@ -12,19 +12,6 @@ using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; -// Get Azure AI Foundry configuration from environment variables -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = System.Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o"; - -const string AssistantInstructions = "You are an AI assistant that helps people find information."; -const string AssistantName = "InformationAssistant"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - [Description("Get the weather for a given location.")] static string GetWeather([Description("The location to get the weather for.")] string location) => $"The weather in {location} is cloudy with a high of 15°C."; @@ -33,14 +20,20 @@ static string GetWeather([Description("The location to get the weather for.")] s static string GetDateTime() => DateTimeOffset.Now.ToString(); +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + AITool dateTimeTool = AIFunctionFactory.Create(GetDateTime, name: nameof(GetDateTime)); AITool getWeatherTool = AIFunctionFactory.Create(GetWeather, name: nameof(GetWeather)); -// Define the agent you want to create. (Prompt Agent in this case) -AIAgent originalAgent = await aiProjectClient.CreateAIAgentAsync( - name: AssistantName, - model: deploymentName, - instructions: AssistantInstructions, +AIAgent originalAgent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are an AI assistant that helps people find information.", + name: "InformationAssistant", tools: [getWeatherTool, dateTimeTool]); // Adding middleware to the agent level @@ -63,24 +56,17 @@ AgentResponse piiResponse = await middlewareEnabledAgent.RunAsync("My name is Jo Console.WriteLine($"Pii filtered response: {piiResponse}"); Console.WriteLine("\n\n=== Example 3: Agent function middleware ==="); - -// Agent function middleware support is limited to agents that wraps a upstream ChatClientAgent or derived from it. - AgentResponse functionCallResponse = await middlewareEnabledAgent.RunAsync("What's the current time and the weather in Seattle?", session); Console.WriteLine($"Function calling response: {functionCallResponse}"); // Special per-request middleware agent. Console.WriteLine("\n\n=== Example 4: Middleware with human in the loop function approval ==="); -AIAgent humanInTheLoopAgent = await aiProjectClient.CreateAIAgentAsync( +AIAgent humanInTheLoopAgent = aiProjectClient.AsAIAgent(deploymentName, + instructions: "You are a Human in the loop testing AI assistant that helps people find information.", name: "HumanInTheLoopAgent", - model: deploymentName, - instructions: "You are an Human in the loop testing AI assistant that helps people find information.", - - // Adding a function with approval required tools: [new ApprovalRequiredAIFunction(AIFunctionFactory.Create(GetWeather, name: nameof(GetWeather)))]); -// Using the ConsolePromptingApprovalMiddleware for a specific request to handle user approval during function calls. AgentResponse response = await humanInTheLoopAgent .AsBuilder() .Use(ConsolePromptingApprovalMiddleware, null) @@ -108,7 +94,6 @@ async ValueTask FunctionCallOverrideWeather(AIAgent agent, FunctionInvo if (context.Function.Name == nameof(GetWeather)) { - // Override the result of the GetWeather function result = "The weather is sunny with a high of 25°C."; } Console.WriteLine($"Function Name: {context!.Function.Name} - Middleware 2 Post-Invoke"); @@ -118,18 +103,16 @@ async ValueTask FunctionCallOverrideWeather(AIAgent agent, FunctionInvo // This middleware redacts PII information from input and output messages. async Task PIIMiddleware(IEnumerable messages, AgentSession? session, AgentRunOptions? options, AIAgent innerAgent, CancellationToken cancellationToken) { - // Redact PII information from input messages var filteredMessages = FilterMessages(messages); Console.WriteLine("Pii Middleware - Filtered Messages Pre-Run"); - var response = await innerAgent.RunAsync(filteredMessages, session, options, cancellationToken).ConfigureAwait(false); + var agentResponse = await innerAgent.RunAsync(filteredMessages, session, options, cancellationToken).ConfigureAwait(false); - // Redact PII information from output messages - response.Messages = FilterMessages(response.Messages); + agentResponse.Messages = FilterMessages(agentResponse.Messages); Console.WriteLine("Pii Middleware - Filtered Messages Post-Run"); - return response; + return agentResponse; static IList FilterMessages(IEnumerable messages) { @@ -138,11 +121,10 @@ async Task PIIMiddleware(IEnumerable messages, Agent static string FilterPii(string content) { - // Regex patterns for PII detection (simplified for demonstration) Regex[] piiPatterns = [ - new(@"\b\d{3}-\d{3}-\d{4}\b", RegexOptions.Compiled), // Phone number (e.g., 123-456-7890) - new(@"\b[\w\.-]+@[\w\.-]+\.\w+\b", RegexOptions.Compiled), // Email address - new(@"\b[A-Z][a-z]+\s[A-Z][a-z]+\b", RegexOptions.Compiled) // Full name (e.g., John Doe) + MyRegex(), + EmailRegex(), + FullNameRegex() ]; foreach (var pattern in piiPatterns) @@ -157,20 +139,17 @@ async Task PIIMiddleware(IEnumerable messages, Agent // This middleware enforces guardrails by redacting certain keywords from input and output messages. async Task GuardrailMiddleware(IEnumerable messages, AgentSession? session, AgentRunOptions? options, AIAgent innerAgent, CancellationToken cancellationToken) { - // Redact keywords from input messages var filteredMessages = FilterMessages(messages); Console.WriteLine("Guardrail Middleware - Filtered messages Pre-Run"); - // Proceed with the agent run - var response = await innerAgent.RunAsync(filteredMessages, session, options, cancellationToken); + var agentResponse = await innerAgent.RunAsync(filteredMessages, session, options, cancellationToken); - // Redact keywords from output messages - response.Messages = FilterMessages(response.Messages); + agentResponse.Messages = FilterMessages(agentResponse.Messages); Console.WriteLine("Guardrail Middleware - Filtered messages Post-Run"); - return response; + return agentResponse; List FilterMessages(IEnumerable messages) { @@ -194,16 +173,13 @@ async Task GuardrailMiddleware(IEnumerable messages, // This middleware handles Human in the loop console interaction for any user approval required during function calling. async Task ConsolePromptingApprovalMiddleware(IEnumerable messages, AgentSession? session, AgentRunOptions? options, AIAgent innerAgent, CancellationToken cancellationToken) { - AgentResponse response = await innerAgent.RunAsync(messages, session, options, cancellationToken); + AgentResponse agentResponse = await innerAgent.RunAsync(messages, session, options, cancellationToken); - // For simplicity, we are assuming here that only function approvals are pending. - List approvalRequests = response.Messages.SelectMany(m => m.Contents).OfType().ToList(); + List approvalRequests = agentResponse.Messages.SelectMany(m => m.Contents).OfType().ToList(); while (approvalRequests.Count > 0) { - // Ask the user to approve each function call request. - // Pass the user input responses back to the agent for further processing. - response.Messages = approvalRequests + agentResponse.Messages = approvalRequests .ConvertAll(functionApprovalRequest => { Console.WriteLine($"The agent would like to invoke the following function, please reply Y to approve: Name {((FunctionCallContent)functionApprovalRequest.ToolCall).Name}"); @@ -211,13 +187,22 @@ async Task ConsolePromptingApprovalMiddleware(IEnumerable m.Contents).OfType().ToList(); + approvalRequests = agentResponse.Messages.SelectMany(m => m.Contents).OfType().ToList(); } - return response; + return agentResponse; } -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(middlewareEnabledAgent.Name); +internal partial class Program +{ + [GeneratedRegex(@"\b\d{3}-\d{3}-\d{4}\b", RegexOptions.Compiled)] + private static partial Regex MyRegex(); + + [GeneratedRegex(@"\b[\w\.-]+@[\w\.-]+\.\w+\b", RegexOptions.Compiled)] + private static partial Regex EmailRegex(); + + [GeneratedRegex(@"\b[A-Z][a-z]+\s[A-Z][a-z]+\b", RegexOptions.Compiled)] + private static partial Regex FullNameRegex(); +} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/README.md new file mode 100644 index 0000000000..26b12a22b8 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step12_Middleware/README.md @@ -0,0 +1,31 @@ +# Middleware with the Responses API + +This sample demonstrates multiple middleware layers working together: PII filtering, guardrails, function invocation logging, and human-in-the-loop approval. + +## What this sample demonstrates + +- Agent-level run middleware (PII filtering, guardrail enforcement) +- Function-level middleware (logging, result overrides) +- Human-in-the-loop approval workflows for sensitive function calls +- Using `.AsBuilder().Use()` to compose middleware +- No server-side agent creation or cleanup required + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\Agent_Step12_Middleware +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/FoundryAgents_Step16_FileSearch.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/Agent_Step13_Plugins.csproj similarity index 91% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/FoundryAgents_Step16_FileSearch.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/Agent_Step13_Plugins.csproj index 4a34560946..1f5e37c1a3 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/FoundryAgents_Step16_FileSearch.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/Agent_Step13_Plugins.csproj @@ -10,13 +10,12 @@ - - + - + diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/Program.cs new file mode 100644 index 0000000000..3af63090c5 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/Program.cs @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use plugins with an AI agent. Plugin classes can +// depend on other services that need to be injected. In this sample, the +// AgentPlugin class uses the WeatherProvider and CurrentTimeProvider classes +// to get weather and current time information. Both services are registered +// in the service collection and injected into the plugin. +// Plugin classes may have many methods, but only some are intended to be used +// as AI functions. The AsAITools method of the plugin class shows how to specify +// which methods should be exposed to the AI agent. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using SampleApp; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +const string AssistantInstructions = "You are a helpful assistant that helps people find information."; +const string AssistantName = "PluginAssistant"; + +// Create a service collection to hold the agent plugin and its dependencies. +ServiceCollection services = new(); +services.AddSingleton(); +services.AddSingleton(); +services.AddSingleton(); // The plugin depends on WeatherProvider and CurrentTimeProvider registered above. + +IServiceProvider serviceProvider = services.BuildServiceProvider(); + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// Create a ChatClientAgent with the options-based constructor to pass services. +AIAgent agent = aiProjectClient.AsAIAgent(new ChatClientAgentOptions +{ + Name = AssistantName, + ChatOptions = new() { ModelId = deploymentName, Instructions = AssistantInstructions, Tools = serviceProvider.GetRequiredService().AsAITools().ToList() } +}, + services: serviceProvider); + +// Invoke the agent and output the text result. +AgentSession session = await agent.CreateSessionAsync(); +Console.WriteLine(await agent.RunAsync("Tell me current time and weather in Seattle.", session)); + +namespace SampleApp +{ + /// + /// The agent plugin that provides weather and current time information. + /// + internal sealed class AgentPlugin + { + private readonly WeatherProvider _weatherProvider; + + /// + /// Initializes a new instance of the class. + /// + /// The weather provider to get weather information. + public AgentPlugin(WeatherProvider weatherProvider) + { + this._weatherProvider = weatherProvider; + } + + /// + /// Gets the weather information for the specified location. + /// + /// + /// This method demonstrates how to use the dependency that was injected into the plugin class. + /// + /// The location to get the weather for. + /// The weather information for the specified location. + public string GetWeather(string location) + { + return this._weatherProvider.GetWeather(location); + } + + /// + /// Gets the current date and time for the specified location. + /// + /// + /// This method demonstrates how to resolve a dependency using the service provider passed to the method. + /// + /// The service provider to resolve the . + /// The location to get the current time for. + /// The current date and time as a . + public DateTimeOffset GetCurrentTime(IServiceProvider sp, string location) + { + CurrentTimeProvider currentTimeProvider = sp.GetRequiredService(); + return currentTimeProvider.GetCurrentTime(location); + } + + /// + /// Returns the functions provided by this plugin. + /// + /// + /// In real world scenarios, a class may have many methods and only a subset of them may be intended to be exposed as AI functions. + /// This method demonstrates how to explicitly specify which methods should be exposed to the AI agent. + /// + /// The functions provided by this plugin. + public IEnumerable AsAITools() + { + yield return AIFunctionFactory.Create(this.GetWeather); + yield return AIFunctionFactory.Create(this.GetCurrentTime); + } + } + + internal sealed class WeatherProvider + { + private readonly string _weatherSummary = "cloudy with a high of 15°C"; + + /// + /// The weather provider that returns weather information. + /// + /// + /// Gets the weather information for the specified location. + /// + /// + /// The weather information is hardcoded for demonstration purposes. + /// In a real application, this could call a weather API to get actual weather data. + /// + /// The location to get the weather for. + /// The weather information for the specified location. + public string GetWeather(string location) + { + return $"The weather in {location} is {this._weatherSummary}."; + } + } + + internal sealed class CurrentTimeProvider + { + private readonly TimeProvider _timeProvider = TimeProvider.System; + + /// + /// Provides the current date and time. + /// + /// + /// This class returns the current date and time using the system's clock. + /// + /// + /// Gets the current date and time. + /// + /// The location to get the current time for (not used in this implementation). + /// The current date and time as a . + public DateTimeOffset GetCurrentTime(string location) + { + return this._timeProvider.GetLocalNow(); + } + } +} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/README.md new file mode 100644 index 0000000000..8cc2f59116 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step13_Plugins/README.md @@ -0,0 +1,29 @@ +# Using Plugins with the Responses API + +This sample shows how to use plugins with a `ChatClientAgent` using the Responses API directly, with dependency injection for plugin services. + +## What this sample demonstrates + +- Creating plugin classes with injected dependencies +- Registering services and building a service provider +- Passing `services` to the `ChatClientAgent` via the options-based constructor +- Using `AIFunctionFactory` to expose plugin methods as AI tools + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/FoundryAgents_Step03_UsingFunctionTools.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/Agent_Step14_CodeInterpreter.csproj similarity index 89% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/FoundryAgents_Step03_UsingFunctionTools.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/Agent_Step14_CodeInterpreter.csproj index daf7e24494..e11688b6ba 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/FoundryAgents_Step03_UsingFunctionTools.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/Agent_Step14_CodeInterpreter.csproj @@ -9,7 +9,6 @@ - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/Program.cs similarity index 59% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/Program.cs index 5a27daed12..8d7f598a4a 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/Program.cs @@ -1,61 +1,31 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to use Code Interpreter Tool with AI Agents. +// This sample shows how to use Code Interpreter Tool with AIProjectClient.AsAIAgent(...). using System.Text; using Azure.AI.Projects; -using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; using OpenAI.Assistants; -using OpenAI.Responses; + +const string AgentInstructions = "You are a personal math tutor. When asked a math question, write and run code using the python tool to answer the question."; +const string AgentName = "CoderAgent-RAPI"; string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; -const string AgentInstructions = "You are a personal math tutor. When asked a math question, write and run code using the python tool to answer the question."; -const string AgentNameMEAI = "CoderAgent-MEAI"; -const string AgentNameNative = "CoderAgent-NATIVE"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Option 1 - Using HostedCodeInterpreterTool + AgentOptions (MEAI + AgentFramework) -// Create the server side agent version -AIAgent agentOption1 = await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - name: AgentNameMEAI, +AIAgent agent = aiProjectClient.AsAIAgent( + deploymentName, instructions: AgentInstructions, + name: AgentName, tools: [new HostedCodeInterpreterTool() { Inputs = [] }]); -// Option 2 - Using PromptAgentDefinition SDK native type -// Create the server side agent version -AIAgent agentOption2 = await aiProjectClient.CreateAIAgentAsync( - name: AgentNameNative, - creationOptions: new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = { - ResponseTool.CreateCodeInterpreterTool( - new CodeInterpreterToolContainer( - CodeInterpreterToolContainerConfiguration.CreateAutomaticContainerConfiguration(fileIds: []) - ) - ), - } - }) -); - -// Either invoke option1 or option2 agent, should have same result -// Option 1 -AgentResponse response = await agentOption1.RunAsync("I need to solve the equation sin(x) + x^2 = 42"); - -// Option 2 -// AgentResponse response = await agentOption2.RunAsync("I need to solve the equation sin(x) + x^2 = 42"); +AgentResponse response = await agent.RunAsync("I need to solve the equation sin(x) + x^2 = 42"); // Get the CodeInterpreterToolCallContent CodeInterpreterToolCallContent? toolCallContent = response.Messages.SelectMany(m => m.Contents).OfType().FirstOrDefault(); @@ -87,7 +57,3 @@ foreach (AIAnnotation annotation in response.Messages.SelectMany(m => m.Contents """); } } - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agentOption1.Name); -await aiProjectClient.Agents.DeleteAgentAsync(agentOption2.Name); diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/README.md new file mode 100644 index 0000000000..1a8cfc8aae --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step14_CodeInterpreter/README.md @@ -0,0 +1,28 @@ +# Code Interpreter with the Responses API + +This sample shows how to use the Code Interpreter tool with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Using `HostedCodeInterpreterTool` with `ChatClientAgent` +- Extracting code input and output from agent responses +- Handling code interpreter annotations and file citations + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Agent_Step15_ComputerUse.csproj similarity index 99% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Agent_Step15_ComputerUse.csproj index 041c72c43e..f739f56123 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/FoundryAgents_Step15_ComputerUse.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Agent_Step15_ComputerUse.csproj @@ -29,5 +29,5 @@ Always - + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_browser_search.png b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Assets/cua_browser_search.png similarity index 100% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_browser_search.png rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Assets/cua_browser_search.png diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_results.png b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Assets/cua_search_results.png similarity index 100% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_results.png rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Assets/cua_search_results.png diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_typed.png b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Assets/cua_search_typed.png similarity index 100% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Assets/cua_search_typed.png rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Assets/cua_search_typed.png diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/ComputerUseUtil.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/ComputerUseUtil.cs similarity index 100% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/ComputerUseUtil.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/ComputerUseUtil.cs diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Program.cs similarity index 61% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Program.cs index 7f6382d085..22f03e27b3 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/Program.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to use Computer Use Tool with AI Agents. +// This sample shows how to use Computer Use Tool with a ChatClientAgent. using Azure.AI.Projects; -using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; using Microsoft.Extensions.AI; using OpenAI.Responses; @@ -15,59 +15,32 @@ internal sealed class Program { private static async Task Main(string[] args) { - string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); - string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "computer-use-preview"; - - // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. - // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid - // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. - // Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. - AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); const string AgentInstructions = @" You are a computer automation assistant. Be direct and efficient. When you reach the search results page, read and describe the actual search result titles and descriptions you can see. "; - const string AgentNameMEAI = "ComputerAgent-MEAI"; - const string AgentNameNative = "ComputerAgent-NATIVE"; + const string AgentName = "ComputerAgent-RAPI"; - // Option 1 - Using ComputerUseTool + AgentOptions (MEAI + AgentFramework) - // Create AIAgent directly - AIAgent agentOption1 = await aiProjectClient.CreateAIAgentAsync( - name: AgentNameMEAI, - model: deploymentName, + string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); + string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "computer-use-preview"; + + // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. + // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid + // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. + AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + + // Create a AIAgent with ComputerUseTool. + AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, instructions: AgentInstructions, + name: AgentName, description: "Computer automation agent with screen interaction capabilities.", tools: [ - ResponseTool.CreateComputerTool(ComputerToolEnvironment.Browser, 1026, 769).AsAITool(), + FoundryAITool.CreateComputerTool(ComputerToolEnvironment.Browser, 1026, 769), ]); - // Option 2 - Using PromptAgentDefinition SDK native type - // Create the server side agent version - AIAgent agentOption2 = await aiProjectClient.CreateAIAgentAsync( - name: AgentNameNative, - creationOptions: new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = { ResponseTool.CreateComputerTool( - environment: new ComputerToolEnvironment("windows"), - displayWidth: 1026, - displayHeight: 769) } - }) - ); - - // Either invoke option1 or option2 agent, should have same result - // Option 1 - await InvokeComputerUseAgentAsync(agentOption1); - - // Option 2 - //await InvokeComputerUseAgentAsync(agentOption2); - - // Cleanup by agent name removes the agent version created. - await aiProjectClient.Agents.DeleteAgentAsync(agentOption1.Name); - await aiProjectClient.Agents.DeleteAgentAsync(agentOption2.Name); + await InvokeComputerUseAgentAsync(agent); } private static async Task InvokeComputerUseAgentAsync(AIAgent agent) @@ -94,10 +67,8 @@ internal sealed class Program // Initial request with screenshot - start with Bing search page Console.WriteLine("Starting computer automation session (initial screenshot: cua_browser_search.png)..."); - // IMPORTANT: Computer-use with the Azure Agents API differs from the vanilla OpenAI Responses API. - // The Azure Agents API rejects requests that include previous_response_id alongside - // computer_call_output items. To work around this, each call uses a fresh session (avoiding - // previous_response_id) and re-sends the full conversation context as input items instead. + // We use PreviousResponseId to chain calls, sending only the new computer_call_output items + // instead of re-sending the full context. AgentSession session = await agent.CreateSessionAsync(); AgentResponse response = await agent.RunAsync(message, session: session, options: runOptions); @@ -161,31 +132,15 @@ internal sealed class Program Console.WriteLine("Sending action result back to agent..."); - // Build the follow-up messages with full conversation context. - // The Azure Agents API rejects previous_response_id when computer_call_output items are - // present, so we must re-send all prior output items (reasoning, computer_call, etc.) - // as input items alongside the computer_call_output to maintain conversation continuity. - List followUpMessages = []; - - // Re-send all response output items as an assistant message so the API has full context - List priorOutputContents = response.Messages - .SelectMany(m => m.Contents) - .ToList(); - followUpMessages.Add(new ChatMessage(ChatRole.Assistant, priorOutputContents)); - - // Add the computer_call_output as a user message + // Send only the computer_call_output — the session carries PreviousResponseId for context continuity. AIContent callOutput = new() { RawRepresentation = new ComputerCallOutputResponseItem( currentCallId, output: ComputerCallOutput.CreateScreenshotOutput(new BinaryData(screenInfo.ImageBytes), "image/png")) }; - followUpMessages.Add(new ChatMessage(ChatRole.User, [callOutput])); - // Create a fresh session so ConversationId does not carry over a previous_response_id. - // Without this, the Azure Agents API returns an error when computer_call_output is present. - session = await agent.CreateSessionAsync(); - response = await agent.RunAsync(followUpMessages, session: session, options: runOptions); + response = await agent.RunAsync([new ChatMessage(ChatRole.User, [callOutput])], session: session, options: runOptions); } } } diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/README.md new file mode 100644 index 0000000000..ecaa18e10f --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step15_ComputerUse/README.md @@ -0,0 +1,29 @@ +# Computer Use with the Responses API + +This sample shows how to use the Computer Use tool with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Using `FoundryAITool.CreateComputerTool()` with `ChatClientAgent` +- Processing computer call actions (click, type, key press) +- Managing the computer use interaction loop with screenshots +- Handling the Azure Agents API workaround for `previous_response_id` with `computer_call_output` + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="computer-use-preview" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/FoundryAgents_Step04_UsingFunctionToolsWithApprovals.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/Agent_Step16_FileSearch.csproj similarity index 89% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/FoundryAgents_Step04_UsingFunctionToolsWithApprovals.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/Agent_Step16_FileSearch.csproj index daf7e24494..e11688b6ba 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/FoundryAgents_Step04_UsingFunctionToolsWithApprovals.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/Agent_Step16_FileSearch.csproj @@ -9,7 +9,6 @@ - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/Program.cs similarity index 69% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/Program.cs index 5371903a9f..c01a951df4 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/Program.cs @@ -1,22 +1,20 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to use File Search Tool with AI Agents. +// This sample shows how to use File Search Tool with a ChatClientAgent. using Azure.AI.Projects; -using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; using OpenAI.Assistants; using OpenAI.Files; -using OpenAI.Responses; string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; const string AgentInstructions = "You are a helpful assistant that can search through uploaded files to answer questions."; -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. +// We need the AIProjectClient to upload files and create vector stores. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. @@ -52,8 +50,11 @@ var vectorStoreResult = await vectorStoresClient.CreateVectorStoreAsync( string vectorStoreId = vectorStoreResult.Value.Id; Console.WriteLine($"Created vector store, vector store ID: {vectorStoreId}"); -AIAgent agent = await CreateAgentWithMEAI(); -// AIAgent agent = await CreateAgentWithNativeSDK(); +// Create a AIAgent with HostedFileSearchTool. +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: AgentInstructions, + name: "FileSearchAgent-RAPI", + tools: [new HostedFileSearchTool() { Inputs = [new HostedVectorStoreContent(vectorStoreId)] }]); // Run the agent Console.WriteLine("\n--- Running File Search Agent ---"); @@ -73,39 +74,9 @@ foreach (AIAnnotation annotation in response.Messages.SelectMany(m => m.Contents } } -// Cleanup. +// Cleanup file resources. Console.WriteLine("\n--- Cleanup ---"); -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); await vectorStoresClient.DeleteVectorStoreAsync(vectorStoreId); await filesClient.DeleteFileAsync(uploadedFile.Id); File.Delete(searchFilePath); Console.WriteLine("Cleanup completed successfully."); - -// --- Agent Creation Options --- - -#pragma warning disable CS8321 // Local function is declared but never used -// Option 1 - Using HostedFileSearchTool (MEAI + AgentFramework) -async Task CreateAgentWithMEAI() -{ - return await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - name: "FileSearchAgent-MEAI", - instructions: AgentInstructions, - tools: [new HostedFileSearchTool() { Inputs = [new HostedVectorStoreContent(vectorStoreId)] }]); -} - -// Option 2 - Using PromptAgentDefinition with ResponseTool.CreateFileSearchTool (Native SDK) -async Task CreateAgentWithNativeSDK() -{ - return await aiProjectClient.CreateAIAgentAsync( - name: "FileSearchAgent-NATIVE", - creationOptions: new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = { - ResponseTool.CreateFileSearchTool(vectorStoreIds: [vectorStoreId]) - } - }) - ); -} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/README.md new file mode 100644 index 0000000000..5ce7f11cda --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step16_FileSearch/README.md @@ -0,0 +1,29 @@ +# File Search with the Responses API + +This sample shows how to use the File Search tool with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Uploading files and creating vector stores via `AIProjectClient` +- Using `HostedFileSearchTool` with `ChatClientAgent` +- Handling file citation annotations in agent responses +- Cleaning up file resources after use + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/FoundryAgents_Step01.1_Basics.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/Agent_Step17_OpenAPITools.csproj similarity index 82% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/FoundryAgents_Step01.1_Basics.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/Agent_Step17_OpenAPITools.csproj index 89b9d8ddc0..4602e9c9e0 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/FoundryAgents_Step01.1_Basics.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/Agent_Step17_OpenAPITools.csproj @@ -6,15 +6,14 @@ enable enable - $(NoWarn);IDE0059 - + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/Program.cs similarity index 55% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/Program.cs index ebf66e6c2c..5cc2720dd9 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/Program.cs @@ -6,16 +6,32 @@ using Azure.AI.Projects; using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; -using OpenAI.Responses; +using Microsoft.Agents.AI.AzureAI; +using Microsoft.Extensions.AI; -// Warning: DefaultAzureCredential is intended for simplicity in development. For production scenarios, consider using a more specific credential. string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; const string AgentInstructions = "You are a helpful assistant that can use the countries API to retrieve information about countries by their currency code."; +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -// A simple OpenAPI specification for the REST Countries API -const string CountriesOpenApiSpec = """ +AITool openApiTool = FoundryAITool.CreateOpenApiTool(CreateOpenAPIFunctionDefinition()); + +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: AgentInstructions, + name: "OpenAPIToolsAgent", + tools: [openApiTool]); + +// Run the agent with a question about countries +Console.WriteLine(await agent.RunAsync("What countries use the Euro (EUR) as their currency? Please list them.")); + +OpenApiFunctionDefinition CreateOpenAPIFunctionDefinition() +{ + // A simple OpenAPI specification for the REST Countries API + const string CountriesOpenApiSpec = """ { "openapi": "3.1.0", "info": { @@ -68,49 +84,12 @@ const string CountriesOpenApiSpec = """ } """; -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Create the OpenAPI function definition -var openApiFunction = new OpenApiFunctionDefinition( - "get_countries", - BinaryData.FromString(CountriesOpenApiSpec), - new OpenAPIAnonymousAuthenticationDetails()) -{ - Description = "Retrieve information about countries by currency code" -}; - -AIAgent agent = await CreateAgentWithMEAI(); -// AIAgent agent = await CreateAgentWithNativeSDK(); - -// Run the agent with a question about countries -Console.WriteLine(await agent.RunAsync("What countries use the Euro (EUR) as their currency? Please list them.")); - -// Cleanup by deleting the agent -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); - -// --- Agent Creation Options --- - -// Option 1 - Using AsAITool wrapping for OpenApiTool (MEAI + AgentFramework) -async Task CreateAgentWithMEAI() -{ - return await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - name: "OpenAPIToolsAgent-MEAI", - instructions: AgentInstructions, - tools: [((ResponseTool)AgentTool.CreateOpenApiTool(openApiFunction)).AsAITool()]); -} - -// Option 2 - Using PromptAgentDefinition with AgentTool.CreateOpenApiTool (Native SDK) -async Task CreateAgentWithNativeSDK() -{ - return await aiProjectClient.CreateAIAgentAsync( - name: "OpenAPIToolsAgent-NATIVE", - creationOptions: new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = { (ResponseTool)AgentTool.CreateOpenApiTool(openApiFunction) } - }) - ); + // Create the OpenAPI function definition + return new( + "get_countries", + BinaryData.FromString(CountriesOpenApiSpec), + new OpenAPIAnonymousAuthenticationDetails()) + { + Description = "Retrieve information about countries by currency code" + }; } diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/README.md new file mode 100644 index 0000000000..52a11ec1dd --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step17_OpenAPITools/README.md @@ -0,0 +1,29 @@ +# OpenAPI Tools with the Responses API + +This sample shows how to use OpenAPI tools with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Defining an OpenAPI specification inline +- Creating an `OpenAPIFunctionDefinition` for the REST Countries API +- Using `FoundryAITool.CreateOpenApiTool()` with `ChatClientAgent` +- Server-side execution of OpenAPI tool calls + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/FoundryAgents_Step01.2_Running.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/Agent_Step18_BingCustomSearch.csproj similarity index 89% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/FoundryAgents_Step01.2_Running.csproj rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/Agent_Step18_BingCustomSearch.csproj index daf7e24494..e11688b6ba 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/FoundryAgents_Step01.2_Running.csproj +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/Agent_Step18_BingCustomSearch.csproj @@ -9,7 +9,6 @@ - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/Program.cs similarity index 54% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/Program.cs index 98ea576226..4ab548403d 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/Program.cs @@ -1,15 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to use Bing Custom Search Tool with AI Agents. +// This sample shows how to use Bing Custom Search Tool with a ChatClientAgent. using Azure.AI.Projects; using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; -using OpenAI.Responses; +using Microsoft.Agents.AI.AzureAI; -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; string connectionId = Environment.GetEnvironmentVariable("AZURE_AI_CUSTOM_SEARCH_CONNECTION_ID") ?? throw new InvalidOperationException("AZURE_AI_CUSTOM_SEARCH_CONNECTION_ID is not set."); string instanceName = Environment.GetEnvironmentVariable("AZURE_AI_CUSTOM_SEARCH_INSTANCE_NAME") ?? throw new InvalidOperationException("AZURE_AI_CUSTOM_SEARCH_INSTANCE_NAME is not set."); @@ -18,19 +16,24 @@ const string AgentInstructions = """ Use the available Bing Custom Search tools to answer questions and perform tasks. """; -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. +// Bing Custom Search tool parameters +BingCustomSearchToolOptions bingCustomSearchToolParameters = new([ + new BingCustomSearchConfiguration(connectionId, instanceName) +]); + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -// Bing Custom Search tool parameters shared by both options -BingCustomSearchToolOptions bingCustomSearchToolParameters = new([ - new BingCustomSearchConfiguration(connectionId, instanceName) -]); - -AIAgent agent = await CreateAgentWithMEAIAsync(); -// AIAgent agent = await CreateAgentWithNativeSDKAsync(); +// Create a AIAgent with Bing Custom Search tool. +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: AgentInstructions, + name: "BingCustomSearchAgent-RAPI", + tools: [FoundryAITool.CreateBingCustomSearchTool(bingCustomSearchToolParameters)]); Console.WriteLine($"Created agent: {agent.Name}"); @@ -42,35 +45,3 @@ foreach (var message in response.Messages) { Console.WriteLine(message.Text); } - -// Cleanup by deleting the agent -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); -Console.WriteLine($"\nDeleted agent: {agent.Name}"); - -// --- Agent Creation Options --- - -// Option 1 - Using AsAITool wrapping for the ResponseTool returned by AgentTool.CreateBingCustomSearchTool (MEAI + AgentFramework) -async Task CreateAgentWithMEAIAsync() -{ - return await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - name: "BingCustomSearchAgent-MEAI", - instructions: AgentInstructions, - tools: [((ResponseTool)AgentTool.CreateBingCustomSearchTool(bingCustomSearchToolParameters)).AsAITool()]); -} - -// Option 2 - Using PromptAgentDefinition with AgentTool.CreateBingCustomSearchTool (Native SDK) -async Task CreateAgentWithNativeSDKAsync() -{ - return await aiProjectClient.CreateAIAgentAsync( - name: "BingCustomSearchAgent-NATIVE", - creationOptions: new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = { - (ResponseTool)AgentTool.CreateBingCustomSearchTool(bingCustomSearchToolParameters), - } - }) - ); -} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/README.md new file mode 100644 index 0000000000..fc48fd8744 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step18_BingCustomSearch/README.md @@ -0,0 +1,36 @@ +# Bing Custom Search with the Responses API + +This sample shows how to use the Bing Custom Search tool with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Configuring `BingCustomSearchToolParameters` with connection ID and instance name +- Using `FoundryAITool.CreateBingCustomSearchTool()` with `ChatClientAgent` +- Processing search results from agent responses + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) +- Bing Custom Search resource configured with a connection ID + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +$env:AZURE_AI_CUSTOM_SEARCH_CONNECTION_ID="your-connection-id" # The full ARM resource URI, e.g., "/subscriptions/.../connections/your-bing-connection" +$env:AZURE_AI_CUSTOM_SEARCH_INSTANCE_NAME="your-instance-name" # The Bing Custom Search configuration name (from Azure portal) +``` + +### Finding the connection ID and instance name + +- **Connection ID** (`AZURE_AI_CUSTOM_SEARCH_CONNECTION_ID`): The full ARM resource URI including the `/projects//connections/` segment. Find the connection name in your Foundry project under **Management center** → **Connected resources**. +- **Instance Name** (`AZURE_AI_CUSTOM_SEARCH_INSTANCE_NAME`): The **configuration name** from your Bing Custom Search resource (Azure portal → your Bing Custom Search resource → **Configurations**). This is _not_ the Azure resource name or the connection name — it's the name of the specific search configuration that defines which domains/sites to search against. + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/Agent_Step19_SharePoint.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/Agent_Step19_SharePoint.csproj new file mode 100644 index 0000000000..e11688b6ba --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/Agent_Step19_SharePoint.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/Program.cs similarity index 53% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/Program.cs index ad6a08abaa..aafca6b8bd 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/Program.cs @@ -1,15 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample shows how to use SharePoint Grounding Tool with AI Agents. +// This sample shows how to use SharePoint Grounding Tool with a ChatClientAgent. using Azure.AI.Projects; using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; -using OpenAI.Responses; +using Microsoft.Agents.AI.AzureAI; -string endpoint = Environment.GetEnvironmentVariable("AZURE_FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_FOUNDRY_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; string sharepointConnectionId = Environment.GetEnvironmentVariable("SHAREPOINT_PROJECT_CONNECTION_ID") ?? throw new InvalidOperationException("SHAREPOINT_PROJECT_CONNECTION_ID is not set."); const string AgentInstructions = """ @@ -17,18 +15,23 @@ const string AgentInstructions = """ Use the available SharePoint tools to answer questions and perform tasks. """; -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. +// Create SharePoint tool options with project connection +var sharepointOptions = new SharePointGroundingToolOptions(); +sharepointOptions.ProjectConnections.Add(new ToolProjectConnection(sharepointConnectionId)); + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -// Create SharePoint tool options with project connection -var sharepointOptions = new SharePointGroundingToolOptions(); -sharepointOptions.ProjectConnections.Add(new ToolProjectConnection(sharepointConnectionId)); - -AIAgent agent = await CreateAgentWithMEAIAsync(); -// AIAgent agent = await CreateAgentWithNativeSDKAsync(); +// Create a AIAgent with SharePoint tool. +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: AgentInstructions, + name: "SharePointAgent-RAPI", + tools: [FoundryAITool.CreateSharepointTool(sharepointOptions)]); Console.WriteLine($"Created agent: {agent.Name}"); @@ -52,33 +55,3 @@ foreach (var message in response.Messages) } } } - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); -Console.WriteLine($"\nDeleted agent: {agent.Name}"); - -// --- Agent Creation Options --- - -// Option 1 - Using AgentTool.CreateSharepointTool + AsAITool() (MEAI + AgentFramework) -async Task CreateAgentWithMEAIAsync() -{ - return await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - name: "SharePointAgent-MEAI", - instructions: AgentInstructions, - tools: [((ResponseTool)AgentTool.CreateSharepointTool(sharepointOptions)).AsAITool()]); -} - -// Option 2 - Using PromptAgentDefinition SDK native type -async Task CreateAgentWithNativeSDKAsync() -{ - return await aiProjectClient.CreateAIAgentAsync( - name: "SharePointAgent-NATIVE", - creationOptions: new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = { AgentTool.CreateSharepointTool(sharepointOptions) } - }) - ); -} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/README.md new file mode 100644 index 0000000000..bfaacee860 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step19_SharePoint/README.md @@ -0,0 +1,30 @@ +# SharePoint Grounding with the Responses API + +This sample shows how to use the SharePoint Grounding tool with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Configuring `SharePointGroundingToolOptions` with project connections +- Using `FoundryAITool.CreateSharepointTool()` with `ChatClientAgent` +- Displaying grounding annotations from agent responses + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) +- SharePoint connection configured in your Microsoft Foundry project + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +$env:SHAREPOINT_PROJECT_CONNECTION_ID="your-sharepoint-connection-id" # The full ARM resource URI, e.g., "/subscriptions/.../connections/SharepointTestTool" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/Agent_Step20_MicrosoftFabric.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/Agent_Step20_MicrosoftFabric.csproj new file mode 100644 index 0000000000..e11688b6ba --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/Agent_Step20_MicrosoftFabric.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/Program.cs new file mode 100644 index 0000000000..7c49afa7ee --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/Program.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use Microsoft Fabric Tool with a ChatClientAgent. + +using Azure.AI.Projects; +using Azure.AI.Projects.Agents; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; + +string fabricConnectionId = Environment.GetEnvironmentVariable("FABRIC_PROJECT_CONNECTION_ID") ?? throw new InvalidOperationException("FABRIC_PROJECT_CONNECTION_ID is not set."); + +const string AgentInstructions = "You are a helpful assistant with access to Microsoft Fabric data. Answer questions based on data available through your Fabric connection."; + +// Configure Microsoft Fabric tool options with project connection +var fabricToolOptions = new FabricDataAgentToolOptions(); +fabricToolOptions.ProjectConnections.Add(new ToolProjectConnection(fabricConnectionId)); + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// Create a AIAgent with Microsoft Fabric tool. +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: AgentInstructions, + name: "FabricAgent-RAPI", + tools: [FoundryAITool.CreateMicrosoftFabricTool(fabricToolOptions)]); + +Console.WriteLine($"Created agent: {agent.Name}"); + +// Run the agent with a sample query +AgentResponse response = await agent.RunAsync("What data is available in the connected Fabric workspace?"); + +Console.WriteLine("\n=== Agent Response ==="); +foreach (var message in response.Messages) +{ + Console.WriteLine(message.Text); +} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/README.md new file mode 100644 index 0000000000..4a4d31e151 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step20_MicrosoftFabric/README.md @@ -0,0 +1,30 @@ +# Microsoft Fabric with the Responses API + +This sample shows how to use the Microsoft Fabric tool with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Configuring `FabricDataAgentToolOptions` with project connections +- Using `FoundryAITool.CreateMicrosoftFabricTool()` with `ChatClientAgent` +- Querying data available through a Fabric connection + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) +- Microsoft Fabric connection configured in your Microsoft Foundry project + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +$env:FABRIC_PROJECT_CONNECTION_ID="your-fabric-connection-id" # The full ARM resource URI, e.g., "/subscriptions/.../connections/FabricTestTool" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/Agent_Step21_WebSearch.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/Agent_Step21_WebSearch.csproj new file mode 100644 index 0000000000..e11688b6ba --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/Agent_Step21_WebSearch.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/Program.cs new file mode 100644 index 0000000000..20d74b0401 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/Program.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use the Web Search Tool with a ChatClientAgent. + +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using OpenAI.Responses; + +const string AgentInstructions = "You are a helpful assistant that can search the web to find current information and answer questions accurately."; +const string AgentName = "WebSearchAgent-RAPI"; + +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// Create a AIAgent with HostedWebSearchTool. +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: AgentInstructions, + name: AgentName, + tools: [new HostedWebSearchTool()]); + +AgentResponse response = await agent.RunAsync("What's the weather today in Seattle?"); + +// Get the text response +Console.WriteLine($"Response: {response.Text}"); + +// Getting any annotations/citations generated by the web search tool +foreach (AIAnnotation annotation in response.Messages.SelectMany(m => m.Contents).SelectMany(c => c.Annotations ?? [])) +{ + Console.WriteLine($"Annotation: {annotation}"); + if (annotation.RawRepresentation is UriCitationMessageAnnotation urlCitation) + { + Console.WriteLine($$""" + Title: {{urlCitation.Title}} + URL: {{urlCitation.Uri}} + """); + } +} diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/README.md new file mode 100644 index 0000000000..1a9d80106d --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step21_WebSearch/README.md @@ -0,0 +1,28 @@ +# Web Search with the Responses API + +This sample shows how to use the Web Search tool with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Using `HostedWebSearchTool` with `ChatClientAgent` +- Processing web search citations and annotations +- Extracting URL citation details (title, URL) from responses + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/Agent_Step22_MemorySearch.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/Agent_Step22_MemorySearch.csproj new file mode 100644 index 0000000000..4602e9c9e0 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/Agent_Step22_MemorySearch.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/Program.cs similarity index 70% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/Program.cs index 60452b7d19..9e1a902f29 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/Program.cs @@ -9,6 +9,8 @@ using Azure.AI.Projects; using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; +using Microsoft.Extensions.AI; using OpenAI.Responses; string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); @@ -22,27 +24,25 @@ const string AgentInstructions = """ When a user shares personal details or preferences, remember them for future conversations. """; -const string AgentNameMEAI = "MemorySearchAgent-MEAI"; -const string AgentNameNative = "MemorySearchAgent-NATIVE"; +const string AgentName = "MemorySearchAgent"; -// Scope identifies the user or context for memory isolation. -// Using a unique user identifier ensures memories are private to that user. string userScope = $"user_{Environment.MachineName}"; -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -DefaultAzureCredential credential = new(); -AIProjectClient aiProjectClient = new(new Uri(endpoint), credential); +MemorySearchPreviewTool memorySearchTool = new(memoryStoreName, userScope) { UpdateDelayInSecs = 0 }; +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +// Create agent using the RAPI path with the MemorySearch tool +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, + instructions: AgentInstructions, + name: AgentName, + tools: [FoundryAITool.FromResponseTool(memorySearchTool)]); // Ensure the memory store exists and has memories to retrieve. await EnsureMemoryStoreAsync(); -// Create the Memory Search tool configuration -MemorySearchPreviewTool memorySearchTool = new(memoryStoreName, userScope) { UpdateDelayInSecs = 0 }; - -// Create agent using Option 1 (MEAI) or Option 2 (Native SDK) -AIAgent agent = await CreateAgentWithMEAI(); -// AIAgent agent = await CreateAgentWithNativeSDK(); - try { Console.WriteLine("Agent created with Memory Search tool. Starting conversation...\n"); @@ -73,41 +73,14 @@ try } finally { - // Cleanup: Delete the agent and memory store. + // Cleanup: Delete the memory store (no server-side agent to clean up in RAPI path). Console.WriteLine("\nCleaning up..."); - await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); - Console.WriteLine("Agent deleted."); await aiProjectClient.MemoryStores.DeleteMemoryStoreAsync(memoryStoreName); Console.WriteLine("Memory store deleted."); } -#pragma warning disable CS8321 // Local function is declared but never used - -// Option 1 - Using MemorySearchTool wrapped as MEAI AITool -async Task CreateAgentWithMEAI() -{ - return await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - name: AgentNameMEAI, - instructions: AgentInstructions, - tools: [((ResponseTool)memorySearchTool).AsAITool()]); -} - -// Option 2 - Using PromptAgentDefinition with MemorySearchTool (Native SDK) -async Task CreateAgentWithNativeSDK() -{ - return await aiProjectClient.CreateAIAgentAsync( - name: AgentNameNative, - creationOptions: new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = { memorySearchTool } - }) - ); -} - // Helpers — kept at the bottom so the main agent flow above stays clean. + async Task EnsureMemoryStoreAsync() { Console.WriteLine($"Creating memory store '{memoryStoreName}'..."); @@ -123,19 +96,37 @@ async Task EnsureMemoryStoreAsync() Console.WriteLine("Memory store created."); } + // Explicitly add memories from a simulated prior conversation. Console.WriteLine("Storing memories from a prior conversation..."); MemoryUpdateOptions memoryOptions = new(userScope) { UpdateDelay = 0 }; - memoryOptions.Items.Add(ResponseItem.CreateUserMessageItem("My name is Alice and I love programming in C#.")); + memoryOptions.Items.Add(ResponseItem.CreateUserMessageItem("My name is Alice and I prefer C#.")); MemoryUpdateResult updateResult = await aiProjectClient.MemoryStores.WaitForMemoriesUpdateAsync( memoryStoreName: memoryStoreName, - pollingInterval: 500, - options: memoryOptions); + options: memoryOptions, + pollingInterval: 500); if (updateResult.Status == MemoryStoreUpdateStatus.Failed) { throw new InvalidOperationException($"Memory update failed: {updateResult.ErrorDetails}"); } - Console.WriteLine($"Memory update completed (status: {updateResult.Status}).\n"); + Console.WriteLine($"Memory update completed (status: {updateResult.Status})."); + + // Quick verification that memories are searchable. + Console.WriteLine("Verifying stored memories..."); + MemorySearchOptions searchOptions = new(userScope) + { + Items = { ResponseItem.CreateUserMessageItem("What are Alice's preferences?") } + }; + MemoryStoreSearchResponse searchResult = await aiProjectClient.MemoryStores.SearchMemoriesAsync( + memoryStoreName: memoryStoreName, + options: searchOptions); + + foreach (var memory in searchResult.Memories) + { + Console.WriteLine($" - {memory.MemoryItem.Content}"); + } + + Console.WriteLine(); } diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/README.md new file mode 100644 index 0000000000..b8020553af --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step22_MemorySearch/README.md @@ -0,0 +1,31 @@ +# Memory Search with the Responses API + +This sample demonstrates how to use the Memory Search tool with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Configuring `MemorySearchPreviewTool` with a memory store and user scope +- Using memory search for cross-conversation recall +- Inspecting `MemorySearchToolCallResponseItem` results +- User profile persistence across conversations + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) +- A memory store created beforehand via Azure Portal or Python SDK + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +$env:AZURE_AI_MEMORY_STORE_ID="your-memory-store-name" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/Agent_Step23_LocalMCP.csproj b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/Agent_Step23_LocalMCP.csproj new file mode 100644 index 0000000000..e51f57c439 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/Agent_Step23_LocalMCP.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/Program.cs b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/Program.cs similarity index 50% rename from dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/Program.cs rename to dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/Program.cs index d41771ef37..c9f8c0060b 100644 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/Program.cs +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/Program.cs @@ -1,24 +1,20 @@ // Copyright (c) Microsoft. All rights reserved. -// This sample demonstrates how to use a local MCP (Model Context Protocol) client with Azure Foundry Agents. -// The MCP tools are resolved locally by connecting directly to the MCP server via HTTP, -// and then passed to the Foundry agent as client-side tools. -// This sample uses the Microsoft Learn MCP endpoint to search documentation. +// This sample demonstrates how to wrap MCP tools with a DelegatingAIFunction to add custom behavior (e.g., logging). +// Compare with Step09 which shows basic MCP tool usage without wrapping. +// The LoggingMcpTool pattern is useful for diagnostics, metering, or adding approval logic around tool calls. using Azure.AI.Projects; using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; using ModelContextProtocol.Client; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; +using SampleApp; const string AgentInstructions = "You are a helpful assistant that can help with Microsoft documentation questions. Use the Microsoft Learn MCP tool to search for documentation."; -const string AgentName = "DocsAgent"; +const string AgentName = "DocsAgent-RAPI"; // Connect to the MCP server locally via HTTP (Streamable HTTP transport). -// The MCP server is hosted at Microsoft Learn and provides documentation search capabilities. Console.WriteLine("Connecting to MCP server at https://learn.microsoft.com/api/mcp ..."); await using McpClient mcpClient = await McpClient.CreateAsync(new HttpClientTransport(new() @@ -34,53 +30,48 @@ Console.WriteLine($"MCP tools available: {string.Join(", ", mcpTools.Select(t => // Wrap each MCP tool with a DelegatingAIFunction to log local invocations. List wrappedTools = mcpTools.Select(tool => (AITool)new LoggingMcpTool(tool)).ToList(); -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. +string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; + // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); -// Create the agent with the locally-resolved MCP tools. -AIAgent agent = await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - name: AgentName, +// Create a AIAgent with the locally-resolved MCP tools. +AIAgent agent = aiProjectClient.AsAIAgent(deploymentName, instructions: AgentInstructions, + name: AgentName, tools: wrappedTools); Console.WriteLine($"Agent '{agent.Name}' created successfully."); -try -{ - // First query - const string Prompt1 = "How does one create an Azure storage account using az cli?"; - Console.WriteLine($"\nUser: {Prompt1}\n"); - AgentResponse response1 = await agent.RunAsync(Prompt1); - Console.WriteLine($"Agent: {response1}"); +// First query +const string Prompt1 = "How does one create an Azure storage account using az cli?"; +Console.WriteLine($"\nUser: {Prompt1}\n"); +AgentResponse response1 = await agent.RunAsync(Prompt1); +Console.WriteLine($"Agent: {response1}"); - Console.WriteLine("\n=======================================\n"); +Console.WriteLine("\n=======================================\n"); - // Second query - const string Prompt2 = "What is Microsoft Agent Framework?"; - Console.WriteLine($"User: {Prompt2}\n"); - AgentResponse response2 = await agent.RunAsync(Prompt2); - Console.WriteLine($"Agent: {response2}"); -} -finally -{ - // Cleanup by removing the agent when done - await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); - Console.WriteLine($"\nAgent '{agent.Name}' deleted."); -} +// Second query +const string Prompt2 = "What is Microsoft Agent Framework?"; +Console.WriteLine($"User: {Prompt2}\n"); +AgentResponse response2 = await agent.RunAsync(Prompt2); +Console.WriteLine($"Agent: {response2}"); -/// -/// Wraps an MCP tool to log when it is invoked locally, -/// confirming that the MCP call is happening client-side. -/// -internal sealed class LoggingMcpTool(AIFunction innerFunction) : DelegatingAIFunction(innerFunction) +namespace SampleApp { - protected override ValueTask InvokeCoreAsync(AIFunctionArguments arguments, CancellationToken cancellationToken) + /// + /// Wraps an MCP tool to log when it is invoked locally, + /// confirming that the MCP call is happening client-side. + /// + internal sealed class LoggingMcpTool(AIFunction innerFunction) : DelegatingAIFunction(innerFunction) { - Console.WriteLine($" >> [LOCAL MCP] Invoking tool '{this.Name}' locally..."); - return base.InvokeCoreAsync(arguments, cancellationToken); + protected override ValueTask InvokeCoreAsync(AIFunctionArguments arguments, CancellationToken cancellationToken) + { + Console.WriteLine($" >> [LOCAL MCP] Invoking tool '{this.Name}' locally..."); + return base.InvokeCoreAsync(arguments, cancellationToken); + } } } diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/README.md new file mode 100644 index 0000000000..84456df2b7 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/Agent_Step23_LocalMCP/README.md @@ -0,0 +1,29 @@ +# Local MCP with the Responses API + +This sample demonstrates how to use a local MCP (Model Context Protocol) client with a `ChatClientAgent` using the Responses API directly. + +## What this sample demonstrates + +- Connecting to an MCP server via HTTP (Streamable HTTP transport) +- Resolving MCP tools locally and wrapping them with logging +- Using `DelegatingAIFunction` to add custom behavior to MCP tools +- Passing locally-resolved MCP tools to `ChatClientAgent` + +## Prerequisites + +- .NET 10 SDK or later +- Microsoft Foundry service endpoint and deployment configured +- Azure CLI installed and authenticated (`az login`) + +Set the following environment variables: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +## Run the sample + +```powershell +dotnet run +``` diff --git a/dotnet/samples/02-agents/AgentsWithFoundry/README.md b/dotnet/samples/02-agents/AgentsWithFoundry/README.md new file mode 100644 index 0000000000..b5802053b3 --- /dev/null +++ b/dotnet/samples/02-agents/AgentsWithFoundry/README.md @@ -0,0 +1,81 @@ +# Getting started with Foundry Agents + +These samples demonstrate how to use Azure AI Foundry with Agent Framework. + +## Quick start + +The simplest way to create a Foundry agent is using the `FoundryAgent` type directly: + +```csharp +FoundryAgent agent = new( + new Uri(endpoint), + new AzureCliCredential(), + model: "gpt-4o-mini", + instructions: "You are good at telling jokes.", + name: "JokerAgent"); + +Console.WriteLine(await agent.RunAsync("Tell me a joke about a pirate.")); +``` + +Or using the `AIProjectClient.AsAIAgent(...)` extensions: + +```csharp +AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); + +FoundryAgent agent = aiProjectClient.AsAIAgent( + model: deploymentName, + instructions: "You are good at telling jokes.", + name: "JokerAgent"); +``` + +## Prerequisites + +- .NET 10 SDK or later +- Foundry project endpoint +- Azure CLI installed and authenticated + +Set: + +```powershell +$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" +``` + +Some samples require extra tool-specific environment variables. See each sample for details. + +## Samples + +| Sample | Description | +| --- | --- | +| [FoundryAgent lifecycle](./Agent_Step00_FoundryAgentLifecycle/) | Create a FoundryAgent directly with endpoint and credentials | +| [Basics (Responses API)](./Agent_Step01_Basics/) | Create and run an agent using AsAIAgent extensions | +| [Multi-turn conversation](./Agent_Step02.1_MultiturnConversation/) | Multi-turn using sessions and response ID chaining | +| [Multi-turn with server conversations](./Agent_Step02.2_MultiturnWithServerConversations/) | Server-side conversations visible in Foundry UI | +| [Using function tools](./Agent_Step03_UsingFunctionTools/) | Function tools | +| [Function tools with approvals](./Agent_Step04_UsingFunctionToolsWithApprovals/) | Human-in-the-loop approval | +| [Structured output](./Agent_Step05_StructuredOutput/) | Structured output with JSON schema | +| [Persisted conversations](./Agent_Step06_PersistedConversations/) | Persisting and resuming conversations | +| [Observability](./Agent_Step07_Observability/) | OpenTelemetry observability | +| [Dependency injection](./Agent_Step08_DependencyInjection/) | DI with a hosted service | +| [Using MCP client as tools](./Agent_Step09_UsingMcpClientAsTools/) | MCP client tools | +| [Using images](./Agent_Step10_UsingImages/) | Image multi-modality | +| [Agent as function tool](./Agent_Step11_AsFunctionTool/) | Agent as a function tool for another | +| [Middleware](./Agent_Step12_Middleware/) | Multiple middleware layers | +| [Plugins](./Agent_Step13_Plugins/) | Plugins with dependency injection | +| [Code interpreter](./Agent_Step14_CodeInterpreter/) | Code interpreter tool | +| [Computer use](./Agent_Step15_ComputerUse/) | Computer use tool | +| [File search](./Agent_Step16_FileSearch/) | File search tool | +| [OpenAPI tools](./Agent_Step17_OpenAPITools/) | OpenAPI tools | +| [Bing custom search](./Agent_Step18_BingCustomSearch/) | Bing Custom Search tool | +| [SharePoint](./Agent_Step19_SharePoint/) | SharePoint grounding tool | +| [Microsoft Fabric](./Agent_Step20_MicrosoftFabric/) | Microsoft Fabric tool | +| [Web search](./Agent_Step21_WebSearch/) | Web search tool | +| [Memory search](./Agent_Step22_MemorySearch/) | Memory search tool | +| [Local MCP](./Agent_Step23_LocalMCP/) | Local MCP client with HTTP transport | + +## Running the samples + +```powershell +cd dotnet/samples/02-agents/AgentsWithFoundry +dotnet run --project .\FoundryAgent_Step01 +``` \ No newline at end of file diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/Program.cs deleted file mode 100644 index 1e1e48d54b..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/Program.cs +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample demonstrates how to use Azure AI Foundry's Red Teaming service to assess -// the safety and resilience of an AI model against adversarial attacks. -// -// It uses the RedTeam API from Azure.AI.Projects to run automated attack simulations -// with various attack strategies (encoding, obfuscation, jailbreaks) across multiple -// risk categories (Violence, HateUnfairness, Sexual, SelfHarm). -// -// For more details, see: -// https://learn.microsoft.com/azure/ai-foundry/concepts/ai-red-teaming-agent - -using Azure.AI.Projects; -using Azure.Identity; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -Console.WriteLine("=" + new string('=', 79)); -Console.WriteLine("RED TEAMING EVALUATION SAMPLE"); -Console.WriteLine("=" + new string('=', 79)); -Console.WriteLine(); - -// Initialize Azure credentials and clients -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -DefaultAzureCredential credential = new(); -AIProjectClient aiProjectClient = new(new Uri(endpoint), credential); - -// Configure the target model for red teaming -AzureOpenAIModelConfiguration targetConfig = new(deploymentName); - -// Create the red team run configuration -RedTeam redTeamConfig = new(targetConfig) -{ - DisplayName = "FinancialAdvisor-RedTeam", - ApplicationScenario = "A financial advisor assistant that provides general financial advice and information.", - NumTurns = 3, - RiskCategories = - { - RiskCategory.Violence, - RiskCategory.HateUnfairness, - RiskCategory.Sexual, - RiskCategory.SelfHarm, - }, - AttackStrategies = - { - AttackStrategy.Easy, - AttackStrategy.Moderate, - AttackStrategy.Jailbreak, - }, -}; - -Console.WriteLine($"Target model: {deploymentName}"); -Console.WriteLine("Risk categories: Violence, HateUnfairness, Sexual, SelfHarm"); -Console.WriteLine("Attack strategies: Easy, Moderate, Jailbreak"); -Console.WriteLine($"Simulation turns: {redTeamConfig.NumTurns}"); -Console.WriteLine(); - -// Submit the red team run to the service -Console.WriteLine("Submitting red team run..."); -RedTeam redTeamRun = await aiProjectClient.RedTeams.CreateAsync(redTeamConfig, options: null); - -Console.WriteLine($"Red team run created: {redTeamRun.Name}"); -Console.WriteLine($"Status: {redTeamRun.Status}"); -Console.WriteLine(); - -// Poll for completion -Console.WriteLine("Waiting for red team run to complete (this may take several minutes)..."); -while (redTeamRun.Status != "Completed" && redTeamRun.Status != "Failed" && redTeamRun.Status != "Canceled") -{ - await Task.Delay(TimeSpan.FromSeconds(15)); - redTeamRun = await aiProjectClient.RedTeams.GetAsync(redTeamRun.Name); - Console.WriteLine($" Status: {redTeamRun.Status}"); -} - -Console.WriteLine(); - -if (redTeamRun.Status == "Completed") -{ - Console.WriteLine("Red team run completed successfully!"); - Console.WriteLine(); - Console.WriteLine("Results:"); - Console.WriteLine(new string('-', 80)); - Console.WriteLine($" Run name: {redTeamRun.Name}"); - Console.WriteLine($" Display name: {redTeamRun.DisplayName}"); - Console.WriteLine($" Status: {redTeamRun.Status}"); - - Console.WriteLine(); - Console.WriteLine("Review the detailed results in the Azure AI Foundry portal:"); - Console.WriteLine($" {endpoint}"); -} -else -{ - Console.WriteLine($"Red team run ended with status: {redTeamRun.Status}"); -} - -Console.WriteLine(); -Console.WriteLine(new string('=', 80)); diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/README.md deleted file mode 100644 index 24e4a62b35..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# Red Teaming with Azure AI Foundry (Classic) - -> [!IMPORTANT] -> This sample uses the **classic Azure AI Foundry** red teaming API (`/redTeams/runs`) via `Azure.AI.Projects`. Results are viewable in the classic Foundry portal experience. The **new Foundry** portal's red teaming feature uses a different evaluation-based API that is not yet available in the .NET SDK. - -This sample demonstrates how to use Azure AI Foundry's Red Teaming service to assess the safety and resilience of an AI model against adversarial attacks. - -## What this sample demonstrates - -- Configuring a red team run targeting an Azure OpenAI model deployment -- Using multiple `AttackStrategy` options (Easy, Moderate, Jailbreak) -- Evaluating across `RiskCategory` categories (Violence, HateUnfairness, Sexual, SelfHarm) -- Submitting a red team scan and polling for completion -- Reviewing results in the Azure AI Foundry portal - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure AI Foundry project (hub and project created) -- Azure OpenAI deployment (e.g., gpt-4o or gpt-4o-mini) -- Azure CLI installed and authenticated (for Azure credential authentication) - -### Regional Requirements - -Red teaming is only available in regions that support risk and safety evaluators: -- **East US 2**, **Sweden Central**, **US North Central**, **France Central**, **Switzerland West** - -### Environment Variables - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-project.services.ai.azure.com/api/projects/your-project" # Replace with your Azure Foundry project endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step01_RedTeaming -dotnet run -``` - -## Expected behavior - -The sample will: - -1. Configure a `RedTeam` run targeting the specified model deployment -2. Define risk categories and attack strategies -3. Submit the scan to Azure AI Foundry's Red Teaming service -4. Poll for completion (this may take several minutes) -5. Display the run status and direct you to the Azure AI Foundry portal for detailed results - -## Understanding Red Teaming - -### Attack Strategies - -| Strategy | Description | -|----------|-------------| -| Easy | Simple encoding/obfuscation attacks (ROT13, Leetspeak, etc.) | -| Moderate | Moderate complexity attacks requiring an LLM for orchestration | -| Jailbreak | Crafted prompts designed to bypass AI safeguards (UPIA) | - -### Risk Categories - -| Category | Description | -|----------|-------------| -| Violence | Content related to violence | -| HateUnfairness | Hate speech or unfair content | -| Sexual | Sexual content | -| SelfHarm | Self-harm related content | - -### Interpreting Results - -- Results are available in the Azure AI Foundry portal (**classic view** — toggle at top-right) under the red teaming section -- Lower Attack Success Rate (ASR) is better — target ASR < 5% for production -- Review individual attack conversations to understand vulnerabilities - -### Current Limitations - -> [!NOTE] -> - The .NET Red Teaming API (`Azure.AI.Projects`) currently supports targeting **model deployments only** via `AzureOpenAIModelConfiguration`. The `AzureAIAgentTarget` type exists in the SDK but is consumed by the **Evaluation Taxonomy** API (`/evaluationtaxonomies`), not by the Red Teaming API (`/redTeams/runs`). -> - Agent-targeted red teaming with agent-specific risk categories (Prohibited actions, Sensitive data leakage, Task adherence) is documented in the [concept docs](https://learn.microsoft.com/azure/ai-foundry/concepts/ai-red-teaming-agent) but is not yet available via the public REST API or .NET SDK. -> - Results from this API appear in the **classic** Azure AI Foundry portal view. The new Foundry portal uses a separate evaluation-based system with `eval_*` identifiers. - -## Related Resources - -- [Azure AI Red Teaming Agent](https://learn.microsoft.com/azure/ai-foundry/concepts/ai-red-teaming-agent) -- [RedTeam .NET API Reference](https://learn.microsoft.com/dotnet/api/azure.ai.projects.redteam?view=azure-dotnet-preview) -- [Risk and Safety Evaluations](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-metrics-built-in#risk-and-safety-evaluators) - -## Next Steps - -After running red teaming: -1. Review attack results and strengthen agent guardrails -2. Explore the Self-Reflection sample (FoundryAgents_Evaluations_Step02_SelfReflection) for quality assessment -3. Set up continuous red teaming in your CI/CD pipeline diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/FoundryAgents_Evaluations_Step02_SelfReflection.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/FoundryAgents_Evaluations_Step02_SelfReflection.csproj deleted file mode 100644 index 646cd75532..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/FoundryAgents_Evaluations_Step02_SelfReflection.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - - - - - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/Program.cs deleted file mode 100644 index 8f8c9fa4ee..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/Program.cs +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample demonstrates how to use Microsoft.Extensions.AI.Evaluation.Quality to evaluate -// an Agent Framework agent's response quality with a self-reflection loop. -// -// It uses GroundednessEvaluator, RelevanceEvaluator, and CoherenceEvaluator to score responses, -// then iteratively asks the agent to improve based on evaluation feedback. -// -// Based on: Reflexion: Language Agents with Verbal Reinforcement Learning (NeurIPS 2023) -// Reference: https://arxiv.org/abs/2303.11366 -// -// For more details, see: -// https://learn.microsoft.com/dotnet/ai/evaluation/libraries - -using Azure.AI.OpenAI; -using Azure.AI.Projects; -using Azure.Identity; -using Microsoft.Agents.AI; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.AI.Evaluation; -using Microsoft.Extensions.AI.Evaluation.Quality; -using Microsoft.Extensions.AI.Evaluation.Safety; - -using ChatMessage = Microsoft.Extensions.AI.ChatMessage; -using ChatRole = Microsoft.Extensions.AI.ChatRole; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; -string openAiEndpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set."); -string evaluatorDeploymentName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOYMENT_NAME") ?? deploymentName; - -Console.WriteLine("=" + new string('=', 79)); -Console.WriteLine("SELF-REFLECTION EVALUATION SAMPLE"); -Console.WriteLine("=" + new string('=', 79)); -Console.WriteLine(); - -// Initialize Azure credentials and client -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -DefaultAzureCredential credential = new(); -AIProjectClient aiProjectClient = new(new Uri(endpoint), credential); - -// Set up the LLM-based chat client for quality evaluators -IChatClient chatClient = new AzureOpenAIClient(new Uri(openAiEndpoint), credential) - .GetChatClient(evaluatorDeploymentName) - .AsIChatClient(); - -// Configure evaluation: quality evaluators use the LLM, safety evaluators use Azure AI Foundry -ContentSafetyServiceConfiguration safetyConfig = new( - credential: credential, - endpoint: new Uri(endpoint)); - -ChatConfiguration chatConfiguration = safetyConfig.ToChatConfiguration( - originalChatConfiguration: new ChatConfiguration(chatClient)); - -// Create a test agent -AIAgent agent = await aiProjectClient.CreateAIAgentAsync( - name: "KnowledgeAgent", - model: deploymentName, - instructions: "You are a helpful assistant. Answer questions accurately based on the provided context."); -Console.WriteLine($"Created agent: {agent.Name}"); -Console.WriteLine(); - -// Example question and grounding context -const string Question = """ - What are the main benefits of using Azure AI Foundry for building AI applications? - """; - -const string Context = """ - Azure AI Foundry is a comprehensive platform for building, deploying, and managing AI applications. - Key benefits include: - 1. Unified development environment with support for multiple AI frameworks and models - 2. Built-in safety and security features including content filtering and red teaming tools - 3. Scalable infrastructure that handles deployment and monitoring automatically - 4. Integration with Azure services like Azure OpenAI, Cognitive Services, and Machine Learning - 5. Evaluation tools for assessing model quality, safety, and performance - 6. Support for RAG (Retrieval-Augmented Generation) patterns with vector search - 7. Enterprise-grade compliance and governance features - """; - -Console.WriteLine("Question:"); -Console.WriteLine(Question); -Console.WriteLine(); - -// Run evaluations -try -{ - await RunSelfReflectionWithGroundedness(agent, Question, Context, chatConfiguration); - await RunQualityEvaluation(agent, Question, Context, chatConfiguration); - await RunCombinedQualityAndSafetyEvaluation(agent, Question, chatConfiguration); -} -finally -{ - // Cleanup - await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); - Console.WriteLine(); - Console.WriteLine("Cleanup: Agent deleted."); -} - -// ============================================================================ -// Implementation Functions -// ============================================================================ - -static async Task RunSelfReflectionWithGroundedness( - AIAgent agent, string question, string context, ChatConfiguration chatConfiguration) -{ - Console.WriteLine("Running Self-Reflection with Groundedness Evaluation..."); - Console.WriteLine(); - - GroundednessEvaluator groundednessEvaluator = new(); - GroundednessEvaluatorContext groundingContext = new(context); - - const int MaxReflections = 3; - double bestScore = 0; - - string currentPrompt = $"Context: {context}\n\nQuestion: {question}"; - - for (int i = 0; i < MaxReflections; i++) - { - Console.WriteLine($"Iteration {i + 1}/{MaxReflections}:"); - Console.WriteLine(new string('-', 40)); - - // Create a new session for each reflection iteration so that - // conversation context does not carry over between runs. This keeps - // each evaluation independent and avoids biasing groundedness scores. - AgentSession session = await agent.CreateSessionAsync(); - AgentResponse agentResponse = await agent.RunAsync(currentPrompt, session); - string responseText = agentResponse.Text; - - Console.WriteLine($"Response: {responseText[..Math.Min(150, responseText.Length)]}..."); - - List messages = - [ - new(ChatRole.User, currentPrompt), - ]; - ChatResponse chatResponse = new(new ChatMessage(ChatRole.Assistant, responseText)); - - EvaluationResult result = await groundednessEvaluator.EvaluateAsync( - messages, - chatResponse, - chatConfiguration, - additionalContext: [groundingContext]); - - NumericMetric groundedness = result.Get(GroundednessEvaluator.GroundednessMetricName); - double score = groundedness.Value ?? 0; - string rating = groundedness.Interpretation?.Rating.ToString() ?? "N/A"; - - Console.WriteLine($"Groundedness score: {score:F1}/5 (Rating: {rating})"); - Console.WriteLine(); - - if (score > bestScore) - { - bestScore = score; - } - - if (score >= 4.0 || i == MaxReflections - 1) - { - if (score >= 4.0) - { - Console.WriteLine("Good groundedness achieved!"); - } - - break; - } - - // Ask for improvement in the next iteration, including the previous response - // so the LLM knows what to improve on (each iteration uses a new session). - currentPrompt = $""" - Context: {context} - - Your previous answer scored {score}/5 on groundedness. - Your previous answer was: - {responseText} - - Please improve your answer to be more grounded in the provided context. - Only include information that is directly supported by the context. - - Question: {question} - """; - Console.WriteLine("Requesting improvement..."); - Console.WriteLine(); - } - - Console.WriteLine($"Best groundedness score: {bestScore:F1}/5"); - Console.WriteLine(new string('=', 80)); - Console.WriteLine(); -} - -static async Task RunQualityEvaluation( - AIAgent agent, string question, string context, ChatConfiguration chatConfiguration) -{ - Console.WriteLine("Running Quality Evaluation (Relevance, Coherence, Groundedness)..."); - Console.WriteLine(); - - IEvaluator[] evaluators = - [ - new RelevanceEvaluator(), - new CoherenceEvaluator(), - new GroundednessEvaluator(), - ]; - - CompositeEvaluator compositeEvaluator = new(evaluators); - GroundednessEvaluatorContext groundingContext = new(context); - - string prompt = $"Context: {context}\n\nQuestion: {question}"; - - AgentSession session = await agent.CreateSessionAsync(); - AgentResponse agentResponse = await agent.RunAsync(prompt, session); - string responseText = agentResponse.Text; - - Console.WriteLine($"Response: {responseText[..Math.Min(150, responseText.Length)]}..."); - Console.WriteLine(); - - List messages = - [ - new(ChatRole.User, prompt), - ]; - ChatResponse chatResponse = new(new ChatMessage(ChatRole.Assistant, responseText)); - - EvaluationResult result = await compositeEvaluator.EvaluateAsync( - messages, - chatResponse, - chatConfiguration, - additionalContext: [groundingContext]); - - foreach (EvaluationMetric metric in result.Metrics.Values) - { - if (metric is NumericMetric n) - { - string rating = n.Interpretation?.Rating.ToString() ?? "N/A"; - Console.WriteLine($" {n.Name,-20} Score: {n.Value:F1}/5 Rating: {rating}"); - } - } - - Console.WriteLine(new string('=', 80)); - Console.WriteLine(); -} - -static async Task RunCombinedQualityAndSafetyEvaluation( - AIAgent agent, string question, ChatConfiguration chatConfiguration) -{ - Console.WriteLine("Running Combined Quality + Safety Evaluation..."); - Console.WriteLine(); - - IEvaluator[] evaluators = - [ - new RelevanceEvaluator(), - new CoherenceEvaluator(), - new ContentHarmEvaluator(), - new ProtectedMaterialEvaluator(), - ]; - - CompositeEvaluator compositeEvaluator = new(evaluators); - - AgentSession session = await agent.CreateSessionAsync(); - AgentResponse agentResponse = await agent.RunAsync(question, session); - string responseText = agentResponse.Text; - - Console.WriteLine($"Response: {responseText[..Math.Min(150, responseText.Length)]}..."); - Console.WriteLine(); - - List messages = - [ - new(ChatRole.User, question), // No context in this evaluation — testing quality and safety on raw question - ]; - ChatResponse chatResponse = new(new ChatMessage(ChatRole.Assistant, responseText)); - - EvaluationResult result = await compositeEvaluator.EvaluateAsync( - messages, - chatResponse, - chatConfiguration); - - Console.WriteLine("Quality Metrics:"); - foreach (EvaluationMetric metric in result.Metrics.Values) - { - if (metric is NumericMetric n) - { - string rating = n.Interpretation?.Rating.ToString() ?? "N/A"; - bool failed = n.Interpretation?.Failed ?? false; - Console.WriteLine($" {n.Name,-25} Score: {n.Value:F1,-6} Rating: {rating,-15} Failed: {failed}"); - } - else if (metric is BooleanMetric b) - { - string rating = b.Interpretation?.Rating.ToString() ?? "N/A"; - bool failed = b.Interpretation?.Failed ?? false; - Console.WriteLine($" {b.Name,-25} Value: {b.Value,-6} Rating: {rating,-15} Failed: {failed}"); - } - } - - Console.WriteLine(new string('=', 80)); -} diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/README.md deleted file mode 100644 index d71eeca6af..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# Self-Reflection Evaluation with Groundedness Assessment - -This sample demonstrates the self-reflection pattern using Agent Framework with `Microsoft.Extensions.AI.Evaluation.Quality` evaluators. The agent iteratively improves its responses based on real groundedness evaluation scores. - -For details on the self-reflection approach, see [Reflexion: Language Agents with Verbal Reinforcement Learning](https://arxiv.org/abs/2303.11366) (NeurIPS 2023). - -## What this sample demonstrates - -- Self-reflection loop that improves responses using real `GroundednessEvaluator` scores -- Using `RelevanceEvaluator` and `CoherenceEvaluator` for multi-metric quality assessment -- Combining quality and safety evaluators with `CompositeEvaluator` -- Configuring `ContentSafetyServiceConfiguration` for safety evaluators alongside LLM-based quality evaluators -- Tracking improvement across iterations - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure AI Foundry project (hub and project created) -- Azure OpenAI deployment (e.g., gpt-4o or gpt-4o-mini) -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -### Azure Resources Required - -1. **Azure AI Hub and Project**: Create these in the Azure Portal - - Follow: https://learn.microsoft.com/azure/ai-foundry/how-to/create-projects -2. **Azure OpenAI Deployment**: Deploy a model (e.g., gpt-4o or gpt-4o-mini) - - Agent model: Used to generate responses - - Evaluator model: Quality evaluators use an LLM; best results with GPT-4o -3. **Azure CLI**: Install and authenticate with `az login` - -### Environment Variables - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-project.api.azureml.ms" # Azure Foundry project endpoint -$env:AZURE_OPENAI_ENDPOINT="https://your-openai.openai.azure.com/" # Azure OpenAI endpoint (for quality evaluators) -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Model deployment name -``` - -**Note**: For best evaluation results, use GPT-4o or GPT-4o-mini as the evaluator model. The groundedness evaluator has been tested and tuned for these models. - -## Run the sample - -Navigate to the sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Evaluations_Step02_SelfReflection -dotnet run -``` - -## Expected behavior - -The sample runs three evaluation scenarios: - -### 1. Self-Reflection with Groundedness -- Asks a question with grounding context -- Evaluates response groundedness using `GroundednessEvaluator` -- If score is below 4/5, asks the agent to improve with feedback -- Repeats up to 3 iterations -- Tracks and reports the best score achieved - -### 2. Quality Evaluation -- Evaluates a single response with multiple quality evaluators: - - `RelevanceEvaluator` — is the response relevant to the question? - - `CoherenceEvaluator` — is the response logically coherent? - - `GroundednessEvaluator` — is the response grounded in the provided context? - -### 3. Combined Quality + Safety Evaluation -- Runs both quality and safety evaluators together: - - `RelevanceEvaluator`, `CoherenceEvaluator` (quality) - - `ContentHarmEvaluator` (safety — violence, hate, sexual, self-harm) - - `ProtectedMaterialEvaluator` (safety — copyrighted content detection) - -## Understanding the Evaluation - -### Groundedness Score (1-5 scale) - -The `GroundednessEvaluator` measures how well the agent's response is grounded in the provided context: - -- **5** = Excellent - Response is fully grounded in context -- **4** = Good - Mostly grounded with minor deviations -- **3** = Fair - Partially grounded but includes unsupported claims -- **2** = Poor - Significant amount of ungrounded content -- **1** = Very Poor - Response is largely unsupported by context - -### Self-Reflection Process - -1. **Initial Response**: Agent generates answer based on question + context -2. **Evaluation**: `GroundednessEvaluator` scores the response (1-5) -3. **Feedback**: If score < 4, agent receives the score and is asked to improve -4. **Iteration**: Process repeats until good score or max iterations - -## Best Practices - -1. **Provide Complete Context**: Ensure grounding context contains all information needed to answer the question -2. **Clear Instructions**: Give the agent clear instructions about staying grounded in context -3. **Use Quality Models**: GPT-4o recommended for evaluation tasks -4. **Multiple Evaluators**: Use combination of evaluators (groundedness + relevance + coherence) -5. **Batch Processing**: For production, process multiple questions in batch - -## Related Resources - -- [Reflexion Paper (NeurIPS 2023)](https://arxiv.org/abs/2303.11366) -- [Microsoft.Extensions.AI.Evaluation Libraries](https://learn.microsoft.com/dotnet/ai/evaluation/libraries) -- [GroundednessEvaluator API Reference](https://learn.microsoft.com/dotnet/api/microsoft.extensions.ai.evaluation.quality.groundednessevaluator) -- [Azure AI Foundry Evaluation Service](https://learn.microsoft.com/azure/ai-foundry/how-to/develop/evaluate-sdk) - -## Next Steps - -After running self-reflection evaluation: -1. Implement similar patterns for other quality metrics (relevance, coherence, fluency) -2. Integrate into CI/CD pipeline for continuous quality assurance -3. Explore the Safety Evaluation sample (FoundryAgents_Evaluations_Step01_RedTeaming) for content safety assessment diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/Program.cs deleted file mode 100644 index f4521d8898..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/Program.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to create and use AI agents with Azure Foundry Agents as the backend. - -using Azure.AI.Projects; -using Azure.AI.Projects.Agents; -using Azure.Identity; -using Microsoft.Agents.AI; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -const string JokerName = "JokerAgent"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Define the agent you want to create. (Prompt Agent in this case) -AgentVersionCreationOptions options = new(new PromptAgentDefinition(model: deploymentName) { Instructions = "You are good at telling jokes." }); - -// Azure.AI.Agents SDK creates and manages agent by name and versions. -// You can create a server side agent version with the Azure.AI.Agents SDK client below. -AgentVersion createdAgentVersion = aiProjectClient.Agents.CreateAgentVersion(agentName: JokerName, options); - -// Note: -// agentVersion.Id = ":", -// agentVersion.Version = , -// agentVersion.Name = - -// You can use an AIAgent with an already created server side agent version. -AIAgent existingJokerAgent = aiProjectClient.AsAIAgent(createdAgentVersion); - -// You can also create another AIAgent version by providing the same name with a different definition/instruction. -AIAgent newJokerAgent = await aiProjectClient.CreateAIAgentAsync(name: JokerName, model: deploymentName, instructions: "You are extremely hilarious at telling jokes."); - -// You can also get the AIAgent latest version by just providing its name. -AIAgent jokerAgentLatest = await aiProjectClient.GetAIAgentAsync(name: JokerName); -AgentVersion latestAgentVersion = jokerAgentLatest.GetService()!; - -// The AIAgent version can be accessed via the GetService method. -Console.WriteLine($"Latest agent version id: {latestAgentVersion.Id}"); - -// Once you have the AIAgent, you can invoke it like any other AIAgent. -Console.WriteLine(await jokerAgentLatest.RunAsync("Tell me a joke about a pirate.")); - -// Cleanup by agent name removes both agent versions created. -await aiProjectClient.Agents.DeleteAgentAsync(existingJokerAgent.Name); diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/README.md deleted file mode 100644 index ce5eca8277..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.1_Basics/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Creating and Managing AI Agents with Versioning - -This sample demonstrates how to create and manage AI agents with Azure Foundry Agents, including: -- Creating agents with different versions -- Retrieving agents by version or latest version -- Running multi-turn conversations with agents -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step01.1_Basics -``` - -## What this sample demonstrates - -1. **Creating agents with versions**: Shows how to create multiple versions of the same agent with different instructions -2. **Retrieving agents**: Demonstrates retrieving agents by specific version or getting the latest version -3. **Multi-turn conversations**: Shows how to use threads to maintain conversation context across multiple agent runs -4. **Agent cleanup**: Demonstrates proper resource cleanup by deleting agents diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/Program.cs deleted file mode 100644 index 0bc17aff0a..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/Program.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to create and use a simple AI agent with Azure Foundry Agents as the backend. - -using Azure.AI.Projects; -using Azure.AI.Projects.Agents; -using Azure.Identity; -using Microsoft.Agents.AI; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -const string JokerInstructions = "You are good at telling jokes."; -const string JokerName = "JokerAgent"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Define the agent you want to create. (Prompt Agent in this case) -AgentVersionCreationOptions options = new(new PromptAgentDefinition(model: deploymentName) { Instructions = JokerInstructions }); - -// Azure.AI.Agents SDK creates and manages agent by name and versions. -// You can create a server side agent version with the Azure.AI.Agents SDK client below. -AgentVersion agentVersion = aiProjectClient.Agents.CreateAgentVersion(agentName: JokerName, options); - -// You can use an AIAgent with an already created server side agent version. -AIAgent jokerAgent = aiProjectClient.AsAIAgent(agentVersion); - -// Invoke the agent with streaming support. -await foreach (AgentResponseUpdate update in jokerAgent.RunStreamingAsync("Tell me a joke about a pirate.")) -{ - Console.WriteLine(update); -} - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(jokerAgent.Name); diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/README.md deleted file mode 100644 index 40cb5e107d..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step01.2_Running/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# Running a Simple AI Agent with Streaming - -This sample demonstrates how to create and run a simple AI agent with Azure Foundry Agents, including both text and streaming responses. - -## What this sample demonstrates - -- Creating a simple AI agent with instructions -- Running an agent with text output -- Running an agent with streaming output -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step01.2_Running -``` - -## Expected behavior - -The sample will: - -1. Create an agent named "JokerAgent" with instructions to tell jokes -2. Run the agent with a text prompt and display the response -3. Run the agent again with streaming to display the response as it's generated -4. Clean up resources by deleting the agent - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/Program.cs deleted file mode 100644 index 7bf12094fc..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/Program.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to create and use a simple AI agent with a multi-turn conversation. - -using Azure.AI.Extensions.OpenAI; -using Azure.AI.Projects; -using Azure.AI.Projects.Agents; -using Azure.Identity; -using Microsoft.Agents.AI; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -const string JokerInstructions = "You are good at telling jokes."; -const string JokerName = "JokerAgent"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Define the agent you want to create. (Prompt Agent in this case) -AgentVersionCreationOptions options = new(new PromptAgentDefinition(model: deploymentName) { Instructions = JokerInstructions }); - -// Retrieve an AIAgent for the created server side agent version. -ChatClientAgent jokerAgent = await aiProjectClient.CreateAIAgentAsync(name: JokerName, options); - -// Invoke the agent with a multi-turn conversation, where the context is preserved in the session object. -// Create a conversation in the server -ProjectConversationsClient conversationsClient = aiProjectClient.GetProjectOpenAIClient().GetProjectConversationsClient(); -ProjectConversation conversation = await conversationsClient.CreateProjectConversationAsync(); - -// Providing the conversation Id is not strictly necessary, but by not providing it no information will show up in the Foundry Project UI as conversations. -// Sessions that don't have a conversation Id will work based on the `PreviousResponseId`. -AgentSession session = await jokerAgent.CreateSessionAsync(conversation.Id); - -Console.WriteLine(await jokerAgent.RunAsync("Tell me a joke about a pirate.", session)); -Console.WriteLine(await jokerAgent.RunAsync("Now add some emojis to the joke and tell it in the voice of a pirate's parrot.", session)); - -// Invoke the agent with a multi-turn conversation and streaming, where the context is preserved in the session object. -session = await jokerAgent.CreateSessionAsync(conversation.Id); -await foreach (AgentResponseUpdate update in jokerAgent.RunStreamingAsync("Tell me a joke about a pirate.", session)) -{ - Console.WriteLine(update); -} -await foreach (AgentResponseUpdate update in jokerAgent.RunStreamingAsync("Now add some emojis to the joke and tell it in the voice of a pirate's parrot.", session)) -{ - Console.WriteLine(update); -} - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(jokerAgent.Name); - -// Cleanup the conversation created. -await conversationsClient.DeleteConversationAsync(conversation.Id); diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/README.md deleted file mode 100644 index 86721bf960..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step02_MultiturnConversation/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Multi-turn Conversation with AI Agents - -This sample demonstrates how to implement multi-turn conversations with AI agents, where context is preserved across multiple agent runs using threads and conversation IDs. - -## What this sample demonstrates - -- Creating an AI agent with instructions -- Creating a project conversation to track conversations in the Foundry UI -- Using threads with conversation IDs to maintain conversation context -- Running multi-turn conversations with text output -- Running multi-turn conversations with streaming output -- Managing agent and conversation lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step02_MultiturnConversation -``` - -## Expected behavior - -The sample will: - -1. Create an agent named "JokerAgent" with instructions to tell jokes -2. Create a project conversation to enable visibility in the Azure Foundry UI -3. Create a thread linked to the conversation ID for context tracking -4. Run the agent with a text prompt and display the response -5. Send a follow-up message to the same thread, demonstrating context preservation -6. Create a new thread sharing the same conversation ID and run the agent with streaming -7. Send a follow-up streaming message to demonstrate multi-turn streaming -8. Clean up resources by deleting the agent and conversation - -## Conversation ID vs PreviousResponseId - -When working with multi-turn conversations, there are two approaches: - -- **With Conversation ID**: By passing a `conversation.Id` to `CreateSessionAsync()`, the conversation will be visible in the Azure Foundry Project UI. This is useful for tracking and debugging conversations. -- **Without Conversation ID**: Sessions created without a conversation ID still work correctly, maintaining context via `PreviousResponseId`. However, these conversations may not appear in the Foundry UI. - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/Program.cs deleted file mode 100644 index cfd74000a6..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/Program.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample demonstrates how to use an agent with function tools. -// It shows both non-streaming and streaming agent interactions using weather-related tools. - -using System.ComponentModel; -using Azure.AI.Projects; -using Azure.Identity; -using Microsoft.Agents.AI; -using Microsoft.Extensions.AI; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -[Description("Get the weather for a given location.")] -static string GetWeather([Description("The location to get the weather for.")] string location) - => $"The weather in {location} is cloudy with a high of 15°C."; - -const string AssistantInstructions = "You are a helpful assistant that can get weather information."; -const string AssistantName = "WeatherAssistant"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Define the agent with function tools. -AITool tool = AIFunctionFactory.Create(GetWeather); - -// Create AIAgent directly -var newAgent = await aiProjectClient.CreateAIAgentAsync(name: AssistantName, model: deploymentName, instructions: AssistantInstructions, tools: [tool]); - -// Getting an already existing agent by name with tools. -/* - * IMPORTANT: Since agents that are stored in the server only know the definition of the function tools (JSON Schema), - * you need to provided all invocable function tools when retrieving the agent so it can invoke them automatically. - * If no invocable tools are provided, the function calling needs to handled manually. - */ -var existingAgent = await aiProjectClient.GetAIAgentAsync(name: AssistantName, tools: [tool]); - -// Non-streaming agent interaction with function tools. -AgentSession session = await existingAgent.CreateSessionAsync(); -Console.WriteLine(await existingAgent.RunAsync("What is the weather like in Amsterdam?", session)); - -// Streaming agent interaction with function tools. -session = await existingAgent.CreateSessionAsync(); -await foreach (AgentResponseUpdate update in existingAgent.RunStreamingAsync("What is the weather like in Amsterdam?", session)) -{ - Console.WriteLine(update); -} - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(existingAgent.Name); diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/README.md deleted file mode 100644 index fa9b5baf21..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step03_UsingFunctionTools/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Using Function Tools with AI Agents - -This sample demonstrates how to use function tools with AI agents, allowing agents to call custom functions to retrieve information. - -## What this sample demonstrates - -- Creating function tools using AIFunctionFactory -- Passing function tools to an AI agent -- Running agents with function tools (text output) -- Running agents with function tools (streaming output) -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step03.1_UsingFunctionTools -``` - -## Expected behavior - -The sample will: - -1. Create an agent named "WeatherAssistant" with a GetWeather function tool -2. Run the agent with a text prompt asking about weather -3. The agent will invoke the GetWeather function tool to retrieve weather information -4. Run the agent again with streaming to display the response as it's generated -5. Clean up resources by deleting the agent - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/README.md deleted file mode 100644 index 42cbd6ba32..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step04_UsingFunctionToolsWithApprovals/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Using Function Tools with Approvals (Human-in-the-Loop) - -This sample demonstrates how to use function tools that require human approval before execution, implementing a human-in-the-loop workflow. - -## What this sample demonstrates - -- Creating approval-required function tools using ApprovalRequiredAIFunction -- Handling user input requests for function approvals -- Implementing human-in-the-loop approval workflows -- Processing agent responses with pending approvals -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step04_UsingFunctionToolsWithApprovals -``` - -## Expected behavior - -The sample will: - -1. Create an agent named "WeatherAssistant" with an approval-required GetWeather function tool -2. Run the agent with a prompt asking about weather -3. The agent will request approval before invoking the GetWeather function -4. The sample will prompt the user to approve or deny the function call (enter 'Y' to approve) -5. After approval, the function will be executed and the result returned to the agent -6. Clean up resources by deleting the agent - -**Note**: For hosted agents with remote users, combine this sample with the Persisted Conversations sample to persist chat history while waiting for user approval. - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/FoundryAgents_Step05_StructuredOutput.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/FoundryAgents_Step05_StructuredOutput.csproj deleted file mode 100644 index daf7e24494..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/FoundryAgents_Step05_StructuredOutput.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/README.md deleted file mode 100644 index 4c44230e18..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step05_StructuredOutput/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Structured Output with AI Agents - -This sample demonstrates how to configure AI agents to produce structured output in JSON format using JSON schemas. - -## What this sample demonstrates - -- Configuring agents with JSON schema response formats -- Using generic RunAsync method for structured output -- Deserializing structured responses into typed objects -- Running agents with streaming and structured output -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step05_StructuredOutput -``` - -## Expected behavior - -The sample will: - -1. Create an agent named "StructuredOutputAssistant" configured to produce JSON output -2. Run the agent with a prompt to extract person information -3. Deserialize the JSON response into a PersonInfo object -4. Display the structured data (Name, Age, Occupation) -5. Run the agent again with streaming and deserialize the streamed JSON response -6. Clean up resources by deleting the agent - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/FoundryAgents_Step06_PersistedConversations.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/FoundryAgents_Step06_PersistedConversations.csproj deleted file mode 100644 index daf7e24494..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/FoundryAgents_Step06_PersistedConversations.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/README.md deleted file mode 100644 index 57a032e9ec..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step06_PersistedConversations/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# Persisted Conversations with AI Agents - -This sample demonstrates how to serialize and persist agent conversation threads to storage, allowing conversations to be resumed later. - -## What this sample demonstrates - -- Serializing agent threads to JSON -- Persisting thread state to disk -- Loading and deserializing thread state from storage -- Resuming conversations with persisted threads -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step06_PersistedConversations -``` - -## Expected behavior - -The sample will: - -1. Create an agent named "JokerAgent" with instructions to tell jokes -2. Create a thread and run the agent with an initial prompt -3. Serialize the thread state to JSON -4. Save the serialized thread to a temporary file -5. Load the thread from the file and deserialize it -6. Resume the conversation with the same thread using a follow-up prompt -7. Clean up resources by deleting the agent - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/README.md deleted file mode 100644 index 459434bce2..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step07_Observability/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Observability with OpenTelemetry - -This sample demonstrates how to add observability to AI agents using OpenTelemetry for tracing and monitoring. - -## What this sample demonstrates - -- Setting up OpenTelemetry TracerProvider -- Configuring console exporter for telemetry output -- Configuring Azure Monitor exporter for Application Insights -- Adding OpenTelemetry middleware to agents -- Running agents with telemetry collection (text and streaming) -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) -- (Optional) Application Insights connection string for Azure Monitor integration - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -$env:APPLICATIONINSIGHTS_CONNECTION_STRING="your-connection-string" # Optional, for Azure Monitor integration -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step07_Observability -``` - -## Expected behavior - -The sample will: - -1. Create a TracerProvider with console exporter (and optionally Azure Monitor exporter) -2. Create an agent named "JokerAgent" with OpenTelemetry middleware -3. Run the agent with a text prompt and display telemetry traces to console -4. Run the agent again with streaming and display telemetry traces -5. Clean up resources by deleting the agent - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/Program.cs deleted file mode 100644 index b7a9874e7b..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/Program.cs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to use dependency injection to register an AIAgent and use it from a hosted service with a user input chat loop. - -using System.ClientModel; -using Azure.AI.Projects; -using Azure.Identity; -using Microsoft.Agents.AI; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -const string JokerInstructions = "You are good at telling jokes."; -const string JokerName = "JokerAgent"; - -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aIProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Create a new agent if one doesn't exist already. -ChatClientAgent agent; -try -{ - agent = await aIProjectClient.GetAIAgentAsync(name: JokerName); -} -catch (ClientResultException ex) when (ex.Status == 404) -{ - agent = await aIProjectClient.CreateAIAgentAsync(name: JokerName, model: deploymentName, instructions: JokerInstructions); -} - -// Create a host builder that we will register services with and then run. -HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - -// Add the agents client to the service collection. -builder.Services.AddSingleton((sp) => aIProjectClient); - -// Add the AI agent to the service collection. -builder.Services.AddSingleton((sp) => agent); - -// Add a sample service that will use the agent to respond to user input. -builder.Services.AddHostedService(); - -// Build and run the host. -using IHost host = builder.Build(); -await host.RunAsync().ConfigureAwait(false); - -/// -/// A sample service that uses an AI agent to respond to user input. -/// -internal sealed class SampleService(AIProjectClient client, AIAgent agent, IHostApplicationLifetime appLifetime) : IHostedService -{ - private AgentSession? _session; - - public async Task StartAsync(CancellationToken cancellationToken) - { - // Create a session that will be used for the entirety of the service lifetime so that the user can ask follow up questions. - this._session = await agent.CreateSessionAsync(cancellationToken); - _ = this.RunAsync(appLifetime.ApplicationStopping); - } - - public async Task RunAsync(CancellationToken cancellationToken) - { - // Delay a little to allow the service to finish starting. - await Task.Delay(100, cancellationToken); - - while (!cancellationToken.IsCancellationRequested) - { - Console.WriteLine("\nAgent: Ask me to tell you a joke about a specific topic. To exit just press Ctrl+C or enter without any input.\n"); - Console.Write("> "); - string? input = Console.ReadLine(); - - // If the user enters no input, signal the application to shut down. - if (string.IsNullOrWhiteSpace(input)) - { - appLifetime.StopApplication(); - break; - } - - // Stream the output to the console as it is generated. - await foreach (AgentResponseUpdate update in agent.RunStreamingAsync(input, this._session, cancellationToken: cancellationToken)) - { - Console.Write(update); - } - - Console.WriteLine(); - } - } - - public async Task StopAsync(CancellationToken cancellationToken) - { - Console.WriteLine("\nDeleting agent ..."); - await client.Agents.DeleteAgentAsync(agent.Name, cancellationToken).ConfigureAwait(false); - } -} diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/README.md deleted file mode 100644 index 12760e736f..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step08_DependencyInjection/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Dependency Injection with AI Agents - -This sample demonstrates how to use dependency injection to register and manage AI agents within a hosted service application. - -## What this sample demonstrates - -- Setting up dependency injection with HostApplicationBuilder -- Registering AIProjectClient as a singleton service -- Registering AIAgent as a singleton service -- Using agents in hosted services -- Interactive chat loop with streaming responses -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step08_DependencyInjection -``` - -## Expected behavior - -The sample will: - -1. Create a host with dependency injection configured -2. Register AIProjectClient and AIAgent as services -3. Create an agent named "JokerAgent" with instructions to tell jokes -4. Start an interactive chat loop where you can ask the agent questions -5. The agent will respond with streaming output -6. Enter an empty line or press Ctrl+C to exit -7. Clean up resources by deleting the agent - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/FoundryAgents_Step09_UsingMcpClientAsTools.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/FoundryAgents_Step09_UsingMcpClientAsTools.csproj deleted file mode 100644 index a6d96cb3db..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/FoundryAgents_Step09_UsingMcpClientAsTools.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - 3afc9b74-af74-4d8e-ae96-fa1c511d11ac - - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/Program.cs deleted file mode 100644 index e1968122a4..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/Program.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to expose an AI agent as an MCP tool. - -using Azure.AI.Projects; -using Azure.Identity; -using Microsoft.Agents.AI; -using Microsoft.Extensions.AI; -using ModelContextProtocol.Client; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -Console.WriteLine("Starting MCP Stdio for @modelcontextprotocol/server-github ... "); - -// Create an MCPClient for the GitHub server -await using var mcpClient = await McpClient.CreateAsync(new StdioClientTransport(new() -{ - Name = "MCPServer", - Command = "npx", - Arguments = ["-y", "--verbose", "@modelcontextprotocol/server-github"], -})); - -// Retrieve the list of tools available on the GitHub server -IList mcpTools = await mcpClient.ListToolsAsync(); -string agentName = "AgentWithMCP"; -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -Console.WriteLine($"Creating the agent '{agentName}' ..."); - -// Define the agent you want to create. (Prompt Agent in this case) -AIAgent agent = await aiProjectClient.CreateAIAgentAsync( - name: agentName, - model: deploymentName, - instructions: "You answer questions related to GitHub repositories only.", - tools: [.. mcpTools.Cast()]); - -string prompt = "Summarize the last four commits to the microsoft/semantic-kernel repository?"; - -Console.WriteLine($"Invoking agent '{agent.Name}' with prompt: {prompt} ..."); - -// Invoke the agent and output the text result. -Console.WriteLine(await agent.RunAsync(prompt)); - -// Clean up the agent after use. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/README.md deleted file mode 100644 index e4e3fe537a..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step09_UsingMcpClientAsTools/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# Using MCP Client Tools with AI Agents - -This sample demonstrates how to use Model Context Protocol (MCP) client tools with AI agents, allowing agents to access tools provided by MCP servers. This sample uses the GitHub MCP server to provide tools for querying GitHub repositories. - -## What this sample demonstrates - -- Creating MCP clients to connect to MCP servers (GitHub server) -- Retrieving tools from MCP servers -- Using MCP tools with AI agents -- Running agents with MCP-provided function tools -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) -- Node.js and npm installed (for running the GitHub MCP server) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step09_UsingMcpClientAsTools -``` - -## Expected behavior - -The sample will: - -1. Start the GitHub MCP server using `@modelcontextprotocol/server-github` -2. Create an MCP client to connect to the GitHub server -3. Retrieve the available tools from the GitHub MCP server -4. Create an agent named "AgentWithMCP" with the GitHub tools -5. Run the agent with a prompt to summarize the last four commits to the microsoft/semantic-kernel repository -6. The agent will use the GitHub MCP tools to query the repository information -7. Clean up resources by deleting the agent \ No newline at end of file diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/FoundryAgents_Step10_UsingImages.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/FoundryAgents_Step10_UsingImages.csproj deleted file mode 100644 index 53661ff199..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/FoundryAgents_Step10_UsingImages.csproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - - - - - - - - - - - - - - Always - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/README.md deleted file mode 100644 index 220104a291..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step10_UsingImages/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Using Images with AI Agents - -This sample demonstrates how to use image multi-modality with an AI agent. It shows how to create a vision-enabled agent that can analyze and describe images using Azure Foundry Agents. - -## What this sample demonstrates - -- Creating a vision-enabled AI agent with image analysis capabilities -- Sending both text and image content to an agent in a single message -- Using `UriContent` for URI-referenced images -- Processing multimodal input (text + image) with an AI agent -- Managing agent lifecycle (creation and deletion) - -## Key features - -- **Vision Agent**: Creates an agent specifically instructed to analyze images -- **Multimodal Input**: Combines text questions with image URI in a single message -- **Azure Foundry Agents Integration**: Uses Azure Foundry Agents with vision capabilities - -## Prerequisites - -Before running this sample, ensure you have: - -1. An Azure OpenAI project set up -2. A compatible model deployment (e.g., gpt-4o) -3. Azure CLI installed and authenticated - -## Environment Variables - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-resource.openai.azure.com/" # Replace with your Azure Foundry Project endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o" # Replace with your model deployment name (optional, defaults to gpt-4o) -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step10_UsingImages -``` - -## Expected behavior - -The sample will: - -1. Create a vision-enabled agent named "VisionAgent" -2. Send a message containing both text ("What do you see in this image?") and a URI-referenced image of a green walkway (nature boardwalk) -3. The agent will analyze the image and provide a description -4. Clean up resources by deleting the agent - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/FoundryAgents_Step11_AsFunctionTool.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/FoundryAgents_Step11_AsFunctionTool.csproj deleted file mode 100644 index 54f37f1aa6..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/FoundryAgents_Step11_AsFunctionTool.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - 3afc9b74-af74-4d8e-ae96-fa1c511d11ac - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/README.md deleted file mode 100644 index 5da59b6edb..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step11_AsFunctionTool/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Using AI Agents as Function Tools (Nested Agents) - -This sample demonstrates how to expose an AI agent as a function tool, enabling nested agent scenarios where one agent can invoke another agent as a tool. - -## What this sample demonstrates - -- Creating an AI agent that can be used as a function tool -- Wrapping an agent as an AIFunction -- Using nested agents where one agent calls another -- Managing multiple agent instances -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step11_AsFunctionTool -``` - -## Expected behavior - -The sample will: - -1. Create a "JokerAgent" that tells jokes -2. Wrap the JokerAgent as a function tool -3. Create a "CoordinatorAgent" that has the JokerAgent as a function tool -4. Run the CoordinatorAgent with a prompt that triggers it to call the JokerAgent -5. The CoordinatorAgent will invoke the JokerAgent as a function tool -6. Clean up resources by deleting both agents - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/README.md deleted file mode 100644 index 96d12d9828..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step12_Middleware/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Agent Middleware - -This sample demonstrates how to add middleware to intercept agent runs and function calls to implement cross-cutting concerns like logging, validation, and guardrails. - -## What This Sample Shows - -1. Azure Foundry Agents integration via `AIProjectClient` and `DefaultAzureCredential` -2. Agent run middleware (logging and monitoring) -3. Function invocation middleware (logging and overriding tool results) -4. Per-request agent run middleware -5. Per-request function pipeline with approval -6. Combining agent-level and per-request middleware - -## Function Invocation Middleware - -Not all agents support function invocation middleware. - -Attempting to use function middleware on agents that do not wrap a ChatClientAgent or derives from it will throw an InvalidOperationException. - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Running the Sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step12_Middleware -``` - -## Expected Behavior - -When you run this sample, you will see the following demonstrations: - -1. **Example 1: Wording Guardrail** - The agent receives a request for harmful content. The guardrail middleware intercepts the request and prevents the agent from responding to harmful prompts, returning a safe response instead. - -2. **Example 2: PII Detection** - The agent receives a message containing personally identifiable information (name, phone number, email). The PII middleware detects and filters this sensitive information before processing. - -3. **Example 3: Agent Function Middleware** - The agent uses function tools (GetDateTime and GetWeather) to answer a question about the current time and weather in Seattle. The function middleware logs the function calls and can override results if needed. - -4. **Example 4: Human-in-the-Loop Function Approval** - The agent attempts to call a weather function, but the approval middleware intercepts the call and prompts the user to approve or deny the function invocation before it executes. The user can respond with "Y" to approve or any other input to deny. - -Each example demonstrates how middleware can be used to implement cross-cutting concerns and control agent behavior at different levels (agent-level and per-request). diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/FoundryAgents_Step13_Plugins.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/FoundryAgents_Step13_Plugins.csproj deleted file mode 100644 index 4a34560946..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/FoundryAgents_Step13_Plugins.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - $(NoWarn);CA1812 - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/Program.cs deleted file mode 100644 index 244d83d632..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/Program.cs +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to use plugins with an AI agent. Plugin classes can -// depend on other services that need to be injected. In this sample, the -// AgentPlugin class uses the WeatherProvider and CurrentTimeProvider classes -// to get weather and current time information. Both services are registered -// in the service collection and injected into the plugin. -// Plugin classes may have many methods, but only some are intended to be used -// as AI functions. The AsAITools method of the plugin class shows how to specify -// which methods should be exposed to the AI agent. - -using Azure.AI.Projects; -using Azure.Identity; -using Microsoft.Agents.AI; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.DependencyInjection; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -const string AssistantInstructions = "You are a helpful assistant that helps people find information."; -const string AssistantName = "PluginAssistant"; - -// Create a service collection to hold the agent plugin and its dependencies. -ServiceCollection services = new(); -services.AddSingleton(); -services.AddSingleton(); -services.AddSingleton(); // The plugin depends on WeatherProvider and CurrentTimeProvider registered above. - -IServiceProvider serviceProvider = services.BuildServiceProvider(); - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Define the agent with plugin tools -// Define the agent you want to create. (Prompt Agent in this case) -AIAgent agent = await aiProjectClient.CreateAIAgentAsync( - name: AssistantName, - model: deploymentName, - instructions: AssistantInstructions, - tools: serviceProvider.GetRequiredService().AsAITools().ToList(), - services: serviceProvider); - -// Invoke the agent and output the text result. -AgentSession session = await agent.CreateSessionAsync(); -Console.WriteLine(await agent.RunAsync("Tell me current time and weather in Seattle.", session)); - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); - -/// -/// The agent plugin that provides weather and current time information. -/// -/// The weather provider to get weather information. -internal sealed class AgentPlugin(WeatherProvider weatherProvider) -{ - /// - /// Gets the weather information for the specified location. - /// - /// - /// This method demonstrates how to use the dependency that was injected into the plugin class. - /// - /// The location to get the weather for. - /// The weather information for the specified location. - public string GetWeather(string location) - { - return weatherProvider.GetWeather(location); - } - - /// - /// Gets the current date and time for the specified location. - /// - /// - /// This method demonstrates how to resolve a dependency using the service provider passed to the method. - /// - /// The service provider to resolve the . - /// The location to get the current time for. - /// The current date and time as a . - public DateTimeOffset GetCurrentTime(IServiceProvider sp, string location) - { - // Resolve the CurrentTimeProvider from the service provider - CurrentTimeProvider currentTimeProvider = sp.GetRequiredService(); - - return currentTimeProvider.GetCurrentTime(location); - } - - /// - /// Returns the functions provided by this plugin. - /// - /// - /// In real world scenarios, a class may have many methods and only a subset of them may be intended to be exposed as AI functions. - /// This method demonstrates how to explicitly specify which methods should be exposed to the AI agent. - /// - /// The functions provided by this plugin. - public IEnumerable AsAITools() - { - yield return AIFunctionFactory.Create(this.GetWeather); - yield return AIFunctionFactory.Create(this.GetCurrentTime); - } -} - -/// -/// The weather provider that returns weather information. -/// -internal sealed class WeatherProvider -{ - /// - /// Gets the weather information for the specified location. - /// - /// - /// The weather information is hardcoded for demonstration purposes. - /// In a real application, this could call a weather API to get actual weather data. - /// - /// The location to get the weather for. - /// The weather information for the specified location. - public string GetWeather(string location) - { - return $"The weather in {location} is cloudy with a high of 15°C."; - } -} - -/// -/// Provides the current date and time. -/// -/// -/// This class returns the current date and time using the system's clock. -/// -internal sealed class CurrentTimeProvider -{ - /// - /// Gets the current date and time. - /// - /// The location to get the current time for (not used in this implementation). - /// The current date and time as a . - public DateTimeOffset GetCurrentTime(string location) - { - return DateTimeOffset.Now; - } -} diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/README.md deleted file mode 100644 index 5c52ffcd1c..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step13_Plugins/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Using Plugins with AI Agents - -This sample demonstrates how to use plugins with AI agents, where plugins are services registered in dependency injection that expose methods as AI function tools. - -## What this sample demonstrates - -- Creating plugin services with methods to expose as tools -- Using AsAITools() to selectively expose plugin methods -- Registering plugins in dependency injection -- Using plugins with AI agents -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step13_Plugins -``` - -## Expected behavior - -The sample will: - -1. Create a plugin service with methods to expose as tools -2. Register the plugin in dependency injection -3. Create an agent named "PluginAgent" with the plugin methods as function tools -4. Run the agent with a prompt that triggers it to call plugin methods -5. The agent will invoke the plugin methods to retrieve information -6. Clean up resources by deleting the agent - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/FoundryAgents_Step14_CodeInterpreter.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/FoundryAgents_Step14_CodeInterpreter.csproj deleted file mode 100644 index 4a34560946..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/FoundryAgents_Step14_CodeInterpreter.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - $(NoWarn);CA1812 - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/README.md deleted file mode 100644 index 34fa18c94c..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step14_CodeInterpreter/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Using Code Interpreter with AI Agents - -This sample demonstrates how to use the code interpreter tool with AI agents. The code interpreter allows agents to write and execute Python code to solve problems, perform calculations, and analyze data. - -## What this sample demonstrates - -- Creating agents with code interpreter capabilities -- Using HostedCodeInterpreterTool (MEAI abstraction) -- Using native SDK code interpreter tools (ResponseTool.CreateCodeInterpreterTool) -- Extracting code inputs and results from agent responses -- Handling code interpreter annotations -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step14_CodeInterpreter -``` - -## Expected behavior - -The sample will: - -1. Create two agents with code interpreter capabilities: - - Option 1: Using HostedCodeInterpreterTool (MEAI abstraction) - - Option 2: Using native SDK code interpreter tools -2. Run the agent with a mathematical problem: "I need to solve the equation sin(x) + x^2 = 42" -3. The agent will use the code interpreter to write and execute Python code to solve the equation -4. Extract and display the code that was executed -5. Display the results from the code execution -6. Display any annotations generated by the code interpreter tool -7. Clean up resources by deleting both agents - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/README.md deleted file mode 100644 index 092f2bd1cf..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step15_ComputerUse/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Using Computer Use Tool with AI Agents - -This sample demonstrates how to use the computer use tool with AI agents. The computer use tool allows agents to interact with a computer environment by viewing the screen, controlling the mouse and keyboard, and performing various actions to help complete tasks. - -> [!NOTE] -> **Azure Agents API vs. vanilla OpenAI Responses API behavior:** -> The Azure Agents API rejects requests that include `previous_response_id` alongside -> `computer_call_output` items — unlike the vanilla OpenAI Responses API, which accepts them. -> This sample works around the limitation by creating a **fresh session for each follow-up call** -> (so no `previous_response_id` is carried over) and re-sending all prior response output items -> (reasoning, computer_call, etc.) as input items to preserve full conversation context. -> Additionally, the sample uses the **current** `CallId` from each computer call response -> (not the initial one) and clears the `ContinuationToken` after polling completes to prevent -> stale tokens from affecting subsequent requests. - -## What this sample demonstrates - -- Creating agents with computer use capabilities -- Using HostedComputerTool (MEAI abstraction) -- Using native SDK computer use tools (ResponseTool.CreateComputerTool) -- Extracting computer action information from agent responses -- Handling computer tool results (text output and screenshots) -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="computer-use-preview" # Optional, defaults to computer-use-preview -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step15_ComputerUse -``` - -## Expected behavior - -The sample will: - -1. Create two agents with computer use capabilities: - - Option 1: Using HostedComputerTool (MEAI abstraction) - - Option 2: Using native SDK computer use tools -2. Run the agent with a task: "I need you to help me search for 'OpenAI news'. Please type 'OpenAI news' and submit the search. Once you see search results, the task is complete." -3. The agent will use the computer use tool to: - - Interpret the screenshots - - Issue action requests based on the task - - Analyze the search results for "OpenAI news" from the screenshots. -4. Extract and display the computer actions performed -5. Display the results from the computer tool execution -6. Display the final response from the agent -7. Clean up resources by deleting both agents diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/README.md deleted file mode 100644 index db74868d3d..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step16_FileSearch/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Using File Search with AI Agents - -This sample demonstrates how to use the file search tool with AI agents. The file search tool allows agents to search through uploaded files stored in vector stores to answer user questions. - -## What this sample demonstrates - -- Uploading files and creating vector stores -- Creating agents with file search capabilities -- Using HostedFileSearchTool (MEAI abstraction) -- Using native SDK file search tools (ResponseTool.CreateFileSearchTool) -- Handling file citation annotations -- Managing agent and resource lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses `DefaultAzureCredential` for authentication. For local development, make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure Identity documentation](https://learn.microsoft.com/dotnet/api/azure.identity.defaultazurecredential). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step16_FileSearch -``` - -## Expected behavior - -The sample will: - -1. Create a temporary text file with employee directory information -2. Upload the file to Azure Foundry -3. Create a vector store with the uploaded file -4. Create an agent with file search capabilities using one of: - - Option 1: Using HostedFileSearchTool (MEAI abstraction) - - Option 2: Using native SDK file search tools -5. Run a query against the agent to search through the uploaded file -6. Display file citation annotations from responses -7. Clean up resources (agent, vector store, and uploaded file) diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/FoundryAgents_Step17_OpenAPITools.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/FoundryAgents_Step17_OpenAPITools.csproj deleted file mode 100644 index 77b76acfa0..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/FoundryAgents_Step17_OpenAPITools.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - $(NoWarn);CA1812;CS8321 - - - - - - - - - - - - - \ No newline at end of file diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/README.md deleted file mode 100644 index a859f6b963..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step17_OpenAPITools/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Using OpenAPI Tools with AI Agents - -This sample demonstrates how to use OpenAPI tools with AI agents. OpenAPI tools allow agents to call external REST APIs defined by OpenAPI specifications. - -## What this sample demonstrates - -- Creating agents with OpenAPI tool capabilities -- Using AgentTool.CreateOpenApiTool with an embedded OpenAPI specification -- Anonymous authentication for public APIs -- Running an agent that can call external REST APIs -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses `DefaultAzureCredential` for authentication, which supports multiple authentication methods including Azure CLI, managed identity, and more. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure Identity documentation](https://learn.microsoft.com/dotnet/api/azure.identity.defaultazurecredential). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step17_OpenAPITools -``` - -## Expected behavior - -The sample will: - -1. Create an agent with an OpenAPI tool configured to call the REST Countries API -2. Ask the agent: "What countries use the Euro (EUR) as their currency?" -3. The agent will use the OpenAPI tool to call the REST Countries API -4. Display the response containing the list of countries that use EUR -5. Clean up resources by deleting the agent \ No newline at end of file diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/FoundryAgents_Step18_BingCustomSearch.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/FoundryAgents_Step18_BingCustomSearch.csproj deleted file mode 100644 index 730d284bd9..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/FoundryAgents_Step18_BingCustomSearch.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - $(NoWarn);CA1812;CS8321 - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/README.md deleted file mode 100644 index ccc1873a04..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step18_BingCustomSearch/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Using Bing Custom Search with AI Agents - -This sample demonstrates how to use the Bing Custom Search tool with AI agents to perform customized web searches. - -## What this sample demonstrates - -- Creating agents with Bing Custom Search capabilities -- Configuring custom search instances via connection ID and instance name -- Two agent creation approaches: MEAI abstraction (Option 1) and Native SDK (Option 2) -- Running search queries through the agent -- Managing agent lifecycle (creation and deletion) - -## Agent creation options - -This sample provides two approaches for creating agents with Bing Custom Search: - -- **Option 1 - MEAI + AgentFramework**: Uses the Agent Framework `ResponseTool` wrapped with `AsAITool()` to call the `CreateAIAgentAsync` overload that accepts `tools:[]`, while still relying on the same underlying Azure AI Projects SDK types as Option 2. -- **Option 2 - Native SDK**: Uses `PromptAgentDefinition` with `AgentVersionCreationOptions` to create the agent directly with the Azure AI Projects SDK types. - -Both options produce the same result. Toggle between them by commenting/uncommenting the corresponding `CreateAgentWith*Async` call in `Program.cs`. - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) -- A Bing Custom Search resource configured in Azure and connected to your Foundry project - -**Note**: This demo uses Azure Default credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. - -Set the following environment variables: - -```powershell -$env:AZURE_FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" -$env:AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -$env:BING_CUSTOM_SEARCH_PROJECT_CONNECTION_ID="/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects//connections/" -$env:BING_CUSTOM_SEARCH_INSTANCE_NAME="your-configuration-name" -``` - -### Finding the connection ID and instance name - -- **Connection ID**: The full ARM resource path including the `/projects//connections/` segment. Find the connection name in your Foundry project under **Management center** → **Connected resources**. -- **Instance Name**: The **configuration name** from the Bing Custom Search resource (Azure portal → your Bing Custom Search resource → **Configurations**). This is _not_ the Azure resource name. - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step18_BingCustomSearch -``` - -## Expected behavior - -The sample will: - -1. Create an agent with Bing Custom Search tool capabilities -2. Run the agent with a search query about Microsoft AI -3. Display the search results returned by the agent -4. Clean up resources by deleting the agent diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/FoundryAgents_Step19_SharePoint.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/FoundryAgents_Step19_SharePoint.csproj deleted file mode 100644 index 4d17fe06bb..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/FoundryAgents_Step19_SharePoint.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - $(NoWarn);CA1812;CS8321 - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/README.md deleted file mode 100644 index ccbd699011..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step19_SharePoint/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# Using SharePoint Grounding with AI Agents - -This sample demonstrates how to use the SharePoint grounding tool with AI agents. The SharePoint grounding tool enables agents to search and retrieve information from SharePoint sites. - -## What this sample demonstrates - -- Creating agents with SharePoint grounding capabilities -- Using AgentTool.CreateSharepointTool (MEAI abstraction) -- Using native SDK SharePoint tools (PromptAgentDefinition) -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure authentication configured for `DefaultAzureCredential` (for example, Azure CLI logged in with `az login`, environment variables, managed identity, or IDE sign-in) -- A SharePoint project connection configured in Azure Foundry - -**Note**: This demo uses `DefaultAzureCredential` for authentication. This credential will try multiple authentication mechanisms in order (such as environment variables, managed identity, Azure CLI login, and IDE sign-in) and use the first one that works. A common option for local development is to sign in with the Azure CLI using `az login` and ensure you have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively) and the [DefaultAzureCredential documentation](https://learn.microsoft.com/dotnet/api/azure.identity.defaultazurecredential). - -Set the following environment variables: - -```powershell -$env:AZURE_FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -$env:SHAREPOINT_PROJECT_CONNECTION_ID="your-sharepoint-connection-id" # Required: SharePoint project connection ID -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step19_SharePoint -``` - -## Expected behavior - -The sample will: - -1. Create two agents with SharePoint grounding capabilities: - - Option 1: Using AgentTool.CreateSharepointTool (MEAI abstraction) - - Option 2: Using native SDK SharePoint tools -2. Run the agent with a query: "List the documents available in SharePoint" -3. The agent will use SharePoint grounding to search and retrieve relevant documents -4. Display the response and any grounding annotations -5. Clean up resources by deleting both agents diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/FoundryAgents_Step20_MicrosoftFabric.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/FoundryAgents_Step20_MicrosoftFabric.csproj deleted file mode 100644 index 4d17fe06bb..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/FoundryAgents_Step20_MicrosoftFabric.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - $(NoWarn);CA1812;CS8321 - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/Program.cs deleted file mode 100644 index e5ab205f68..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/Program.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to use Microsoft Fabric Tool with AI Agents. - -using Azure.AI.Projects; -using Azure.AI.Projects.Agents; -using Azure.Identity; -using Microsoft.Agents.AI; -using OpenAI.Responses; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_FOUNDRY_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; -string fabricConnectionId = Environment.GetEnvironmentVariable("FABRIC_PROJECT_CONNECTION_ID") ?? throw new InvalidOperationException("FABRIC_PROJECT_CONNECTION_ID is not set."); - -const string AgentInstructions = "You are a helpful assistant with access to Microsoft Fabric data. Answer questions based on data available through your Fabric connection."; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. -// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid -// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Configure Microsoft Fabric tool options with project connection -var fabricToolOptions = new FabricDataAgentToolOptions(); -fabricToolOptions.ProjectConnections.Add(new ToolProjectConnection(fabricConnectionId)); - -AIAgent agent = await CreateAgentWithMEAIAsync(); -// AIAgent agent = await CreateAgentWithNativeSDKAsync(); - -Console.WriteLine($"Created agent: {agent.Name}"); - -// Run the agent with a sample query -AgentResponse response = await agent.RunAsync("What data is available in the connected Fabric workspace?"); - -Console.WriteLine("\n=== Agent Response ==="); -foreach (var message in response.Messages) -{ - Console.WriteLine(message.Text); -} - -// Cleanup by deleting the agent -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); -Console.WriteLine($"\nDeleted agent: {agent.Name}"); - -// --- Agent Creation Options --- - -// Option 1 - Using AsAITool wrapping for the ResponseTool returned by AgentTool.CreateMicrosoftFabricTool (MEAI + AgentFramework) -async Task CreateAgentWithMEAIAsync() -{ - return await aiProjectClient.CreateAIAgentAsync( - model: deploymentName, - name: "FabricAgent-MEAI", - instructions: AgentInstructions, - tools: [((ResponseTool)AgentTool.CreateMicrosoftFabricTool(fabricToolOptions)).AsAITool()]); -} - -// Option 2 - Using PromptAgentDefinition with AgentTool.CreateMicrosoftFabricTool (Native SDK) -async Task CreateAgentWithNativeSDKAsync() -{ - return await aiProjectClient.CreateAIAgentAsync( - name: "FabricAgent-NATIVE", - creationOptions: new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = - { - AgentTool.CreateMicrosoftFabricTool(fabricToolOptions), - } - }) - ); -} diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/README.md deleted file mode 100644 index a5faf79d9d..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step20_MicrosoftFabric/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Using Microsoft Fabric Tool with AI Agents - -This sample demonstrates how to use the Microsoft Fabric tool with AI Agents, allowing agents to query and interact with data in Microsoft Fabric workspaces. - -## What this sample demonstrates - -- Creating agents with Microsoft Fabric data access capabilities -- Using FabricDataAgentToolOptions to configure Fabric connections -- Two agent creation approaches: MEAI abstraction (Option 1) and Native SDK (Option 2) -- Managing agent lifecycle (creation and deletion) - -## Agent creation options - -This sample provides two approaches for creating agents with Microsoft Fabric: - -- **Option 1 - MEAI + AgentFramework**: Uses the Agent Framework `ResponseTool` wrapped with `AsAITool()` to call the `CreateAIAgentAsync` overload that accepts `tools:[]`, while still relying on the same underlying Azure AI Projects SDK types as Option 2. -- **Option 2 - Native SDK**: Uses `PromptAgentDefinition` with `AgentVersionCreationOptions` to create the agent directly with the Azure AI Projects SDK types. - -Both options produce the same result. Toggle between them by commenting/uncommenting the corresponding `CreateAgentWith*Async` call in `Program.cs`. - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) -- A Microsoft Fabric workspace with a configured project connection in Azure Foundry - -**Note**: This demo uses Azure Default credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. - -Set the following environment variables: - -```powershell -$env:AZURE_FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" -$env:AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -$env:FABRIC_PROJECT_CONNECTION_ID="your-fabric-connection-id" # The Fabric project connection ID from Azure Foundry -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step20_MicrosoftFabric -``` - -## Expected behavior - -The sample will: - -1. Create an agent with Microsoft Fabric tool capabilities -2. Configure the agent with a Fabric project connection -3. Run the agent with a query about available Fabric data -4. Display the agent's response -5. Clean up resources by deleting the agent diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/FoundryAgents_Step21_WebSearch.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/FoundryAgents_Step21_WebSearch.csproj deleted file mode 100644 index 4d17fe06bb..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/FoundryAgents_Step21_WebSearch.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - $(NoWarn);CA1812;CS8321 - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/Program.cs b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/Program.cs deleted file mode 100644 index c116a975e1..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/Program.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to use the Responses API Web Search Tool with AI Agents. - -using Azure.AI.Projects; -using Azure.AI.Projects.Agents; -using Azure.Identity; -using Microsoft.Agents.AI; -using Microsoft.Extensions.AI; -using OpenAI.Responses; - -string endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); -string deploymentName = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4o-mini"; - -const string AgentInstructions = "You are a helpful assistant that can search the web to find current information and answer questions accurately."; -const string AgentName = "WebSearchAgent"; - -// Get a client to create/retrieve/delete server side agents with Azure Foundry Agents. -AIProjectClient aiProjectClient = new(new Uri(endpoint), new DefaultAzureCredential()); - -// Option 1 - Using HostedWebSearchTool (MEAI + AgentFramework) -AIAgent agent = await CreateAgentWithMEAIAsync(); - -// Option 2 - Using PromptAgentDefinition with the Responses API native type -// AIAgent agent = await CreateAgentWithNativeSDKAsync(); - -AgentResponse response = await agent.RunAsync("What's the weather today in Seattle?"); - -// Get the text response -Console.WriteLine($"Response: {response.Text}"); - -// Getting any annotations/citations generated by the web search tool -foreach (AIAnnotation annotation in response.Messages.SelectMany(m => m.Contents).SelectMany(c => c.Annotations ?? [])) -{ - Console.WriteLine($"Annotation: {annotation}"); - if (annotation.RawRepresentation is UriCitationMessageAnnotation urlCitation) - { - Console.WriteLine($$""" - Title: {{urlCitation.Title}} - URL: {{urlCitation.Uri}} - """); - } -} - -// Cleanup by agent name removes the agent version created. -await aiProjectClient.Agents.DeleteAgentAsync(agent.Name); - -// Creates the agent using the HostedWebSearchTool MEAI abstraction that maps to the built-in Responses API web search tool. -async Task CreateAgentWithMEAIAsync() - => await aiProjectClient.CreateAIAgentAsync( - name: AgentName, - model: deploymentName, - instructions: AgentInstructions, - tools: [new HostedWebSearchTool()]); - -// Creates the agent using the PromptAgentDefinition with the Responses API native ResponseTool.CreateWebSearchTool(). -async Task CreateAgentWithNativeSDKAsync() - => await aiProjectClient.CreateAIAgentAsync( - AgentName, - new AgentVersionCreationOptions( - new PromptAgentDefinition(model: deploymentName) - { - Instructions = AgentInstructions, - Tools = { ResponseTool.CreateWebSearchTool() } - })); diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/README.md deleted file mode 100644 index 8da390878c..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step21_WebSearch/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Using Web Search with AI Agents - -This sample demonstrates how to use the Responses API web search tool with AI agents. The web search tool allows agents to search the web for current information to answer questions accurately. - -## What this sample demonstrates - -- Creating agents with web search capabilities -- Using HostedWebSearchTool (MEAI abstraction) -- Using native SDK web search tools (ResponseTool.CreateWebSearchTool) -- Extracting text responses and URL citations from agent responses -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure authentication configured for `DefaultAzureCredential` (for example, Azure CLI logged in with `az login`, environment variables, managed identity, or IDE sign-in) - -**Note**: This sample authenticates using `DefaultAzureCredential` from the Azure Identity library, which will try several credential sources (including Azure CLI, environment variables, managed identity, and IDE sign-in). Ensure at least one supported credential source is available. For more information, see the [Azure Identity documentation](https://learn.microsoft.com/dotnet/api/overview/azure/identity-readme). - -**Note**: The web search tool uses the built-in web search capability from the OpenAI Responses API. - -Set the following environment variables: - -```powershell -$env:AZURE_FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step21_WebSearch -``` - -## Expected behavior - -The sample will: - -1. Create an agent with web search capabilities using HostedWebSearchTool (MEAI abstraction) - - Alternative: Using native SDK web search tools (commented out in code) - - Alternative: Retrieving an existing agent by name (commented out in code) -2. Run the agent with a query: "What's the weather today in Seattle?" -3. The agent will use the web search tool to find current information -4. Display the text response from the agent -5. Display any URL citations from web search results -6. Clean up resources by deleting the agent diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/FoundryAgents_Step22_MemorySearch.csproj b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/FoundryAgents_Step22_MemorySearch.csproj deleted file mode 100644 index d83a9d9202..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/FoundryAgents_Step22_MemorySearch.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - $(NoWarn);CA1812 - - - - - - - - - - - - - diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/README.md deleted file mode 100644 index 9e6d79d579..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step22_MemorySearch/README.md +++ /dev/null @@ -1,92 +0,0 @@ -# Using Memory Search with AI Agents - -This sample demonstrates how to use the Memory Search tool with AI agents. The Memory Search tool enables agents to recall information from previous conversations, supporting user profile persistence and chat summaries across sessions. - -## What this sample demonstrates - -- Creating an agent with Memory Search tool capabilities -- Configuring memory scope for user isolation -- Having conversations where the agent remembers past information -- Inspecting memory search results from agent responses -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) -- **A pre-created Memory Store** (see below) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -### Creating a Memory Store - -Memory stores must be created before running this sample. The .NET SDK currently only supports **using** existing memory stores with agents. To create a memory store, use one of these methods: - -**Option 1: Azure Portal** -1. Navigate to your Azure AI Foundry project -2. Go to the Memory section -3. Create a new memory store with your desired settings - -**Option 2: Python SDK** -```python -from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import MemoryStoreDefaultDefinition, MemoryStoreDefaultOptions -from azure.identity import DefaultAzureCredential - -project_client = AIProjectClient( - endpoint="https://your-endpoint.openai.azure.com/", - credential=DefaultAzureCredential() -) - -memory_store = await project_client.memory_stores.create( - name="my-memory-store", - description="Memory store for Agent Framework conversations", - definition=MemoryStoreDefaultDefinition( - chat_model=os.environ["AZURE_AI_CHAT_MODEL_DEPLOYMENT_NAME"], - embedding_model=os.environ["AZURE_AI_EMBEDDING_MODEL_DEPLOYMENT_NAME"], - options=MemoryStoreDefaultOptions( - user_profile_enabled=True, - chat_summary_enabled=True - ) - ) -) -``` - -## Environment Variables - -Set the following environment variables: - -```powershell -$env:AZURE_FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" -$env:AZURE_FOUNDRY_PROJECT_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -$env:AZURE_AI_MEMORY_STORE_NAME="your-memory-store-name" # Required - name of pre-created memory store -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step22_MemorySearch -``` - -## Expected behavior - -The sample will: - -1. Create an agent with Memory Search tool configured -2. Send a message with personal information ("My name is Alice and I love programming in C#") -3. Wait for memory indexing -4. Ask the agent to recall the previously shared information -5. Display memory search results if available in the response -6. Clean up by deleting the agent (note: memory store persists) - -## Important notes - -- **Memory Store Lifecycle**: Memory stores are long-lived resources and are NOT deleted when the agent is deleted. Clean them up separately via Azure Portal or Python SDK. -- **Scope**: The `scope` parameter isolates memories per user/context. Use unique identifiers for different users. -- **Update Delay**: The `UpdateDelay` parameter controls how quickly new memories are indexed. diff --git a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/README.md b/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/README.md deleted file mode 100644 index 8651108987..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/FoundryAgents_Step23_LocalMCP/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Using Local MCP Client with Azure Foundry Agents - -This sample demonstrates how to use a local MCP (Model Context Protocol) client with Azure Foundry Agents. Unlike the hosted MCP approach where Azure Foundry invokes the MCP server on the service side, this sample connects to the MCP server directly from the client via HTTP (Streamable HTTP transport) and passes the resolved tools to the agent. - -## What this sample demonstrates - -- Connecting to an MCP server locally using `HttpClientTransport` -- Discovering available tools from the MCP server client-side -- Passing locally-resolved MCP tools to a Foundry agent -- Using the Microsoft Learn MCP endpoint for documentation search -- Managing agent lifecycle (creation and deletion) - -## Prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and deployment configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: This demo uses Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -## Run the sample - -Navigate to the FoundryAgents sample directory and run: - -```powershell -cd dotnet/samples/02-agents/FoundryAgents -dotnet run --project .\FoundryAgents_Step23_LocalMCP -``` - -## Expected behavior - -The sample will: - -1. Connect to the Microsoft Learn MCP server via HTTP and list available tools -2. Create an agent with the locally-resolved MCP tools -3. Ask two questions about Microsoft documentation -4. The agent will use the MCP tools (invoked locally) to search Microsoft Learn documentation -5. Display the agent's responses with information from the documentation -6. Clean up resources by deleting the agent diff --git a/dotnet/samples/02-agents/FoundryAgents/README.md b/dotnet/samples/02-agents/FoundryAgents/README.md deleted file mode 100644 index 426a8cdad5..0000000000 --- a/dotnet/samples/02-agents/FoundryAgents/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# Getting started with Foundry Agents - -The getting started with Foundry Agents samples demonstrate the fundamental concepts and functionalities -of Azure Foundry Agents and can be used with Azure Foundry as the AI provider. - -These samples showcase how to work with agents managed through Azure Foundry, including agent creation, -versioning, multi-turn conversations, and advanced features like code interpretation and computer use. - -## Classic vs New Foundry Agents - -> [!NOTE] -> Recently, Azure Foundry introduced a new and improved experience for creating and managing AI agents, which is the target of these samples. - -For more information about the previous classic agents and for what's new in Foundry Agents, see the [Foundry Agents migration documentation](https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/migrate?view=foundry). - -For a sample demonstrating how to use classic Foundry Agents, see the following: [Agent with Azure AI Persistent](../AgentProviders/Agent_With_AzureAIAgentsPersistent/README.md). - -## Agent Versioning and Static Definitions - -One of the key architectural changes in the new Foundry Agents compared to the classic experience is how agent definitions are handled. In the new architecture, agents have **versions** and their definitions are established at creation time. This means that the agent's configuration—including instructions, tools, and options—is fixed when the agent version is created. - -> [!IMPORTANT] -> Agent versions are static and strictly adhere to their original definition. Any attempt to provide or override tools, instructions, or options during an agent run or request will be ignored by the agent, as the API does not support runtime configuration changes. All agent behavior must be defined at agent creation time. - -This design ensures consistency and predictability in agent behavior across all interactions with a specific agent version. - -The Agent Framework intentionally ignores unsupported runtime parameters rather than throwing exceptions. This abstraction-first approach ensures that code written against the unified agent abstraction remains portable across providers (OpenAI, Azure OpenAI, Foundry Agents). It removes the need for provider-specific conditional logic. Teams can adopt Foundry Agents without rewriting existing orchestration code. Configurations that work with other providers will gracefully degrade, rather than fail, when the underlying API does not support them. - -## Getting started with Foundry Agents prerequisites - -Before you begin, ensure you have the following prerequisites: - -- .NET 10 SDK or later -- Azure Foundry service endpoint and project configured -- Azure CLI installed and authenticated (for Azure credential authentication) - -**Note**: These samples use Azure Foundry Agents. For more information, see [Azure AI Foundry documentation](https://learn.microsoft.com/en-us/azure/ai-foundry/). - -**Note**: These samples use Azure CLI credentials for authentication. Make sure you're logged in with `az login` and have access to the Azure Foundry resource. For more information, see the [Azure CLI documentation](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively). - -## Samples - -|Sample|Description| -|---|---| -|[Basics](./FoundryAgents_Step01.1_Basics/)|This sample demonstrates how to create and manage AI agents with versioning| -|[Running a simple agent](./FoundryAgents_Step01.2_Running/)|This sample demonstrates how to create and run a basic Foundry agent| -|[Multi-turn conversation](./FoundryAgents_Step02_MultiturnConversation/)|This sample demonstrates how to implement a multi-turn conversation with a Foundry agent| -|[Using function tools](./FoundryAgents_Step03_UsingFunctionTools/)|This sample demonstrates how to use function tools with a Foundry agent| -|[Using function tools with approvals](./FoundryAgents_Step04_UsingFunctionToolsWithApprovals/)|This sample demonstrates how to use function tools where approvals require human in the loop approvals before execution| -|[Structured output](./FoundryAgents_Step05_StructuredOutput/)|This sample demonstrates how to use structured output with a Foundry agent| -|[Persisted conversations](./FoundryAgents_Step06_PersistedConversations/)|This sample demonstrates how to persist conversations and reload them later| -|[Observability](./FoundryAgents_Step07_Observability/)|This sample demonstrates how to add telemetry to a Foundry agent| -|[Dependency injection](./FoundryAgents_Step08_DependencyInjection/)|This sample demonstrates how to add and resolve a Foundry agent with a dependency injection container| -|[Using MCP client as tools](./FoundryAgents_Step09_UsingMcpClientAsTools/)|This sample demonstrates how to use MCP clients as tools with a Foundry agent| -|[Using images](./FoundryAgents_Step10_UsingImages/)|This sample demonstrates how to use image multi-modality with a Foundry agent| -|[Exposing as a function tool](./FoundryAgents_Step11_AsFunctionTool/)|This sample demonstrates how to expose a Foundry agent as a function tool| -|[Using middleware](./FoundryAgents_Step12_Middleware/)|This sample demonstrates how to use middleware with a Foundry agent| -|[Using plugins](./FoundryAgents_Step13_Plugins/)|This sample demonstrates how to use plugins with a Foundry agent| -|[Code interpreter](./FoundryAgents_Step14_CodeInterpreter/)|This sample demonstrates how to use the code interpreter tool with a Foundry agent| -|[Computer use](./FoundryAgents_Step15_ComputerUse/)|This sample demonstrates how to use computer use capabilities with a Foundry agent| -|[File search](./FoundryAgents_Step16_FileSearch/)|This sample demonstrates how to use the file search tool with a Foundry agent| -|[OpenAPI tools](./FoundryAgents_Step17_OpenAPITools/)|This sample demonstrates how to use OpenAPI tools with a Foundry agent| -|[Bing Custom Search](./FoundryAgents_Step18_BingCustomSearch/)|This sample demonstrates how to use Bing Custom Search tool with a Foundry agent| -|[SharePoint grounding](./FoundryAgents_Step19_SharePoint/)|This sample demonstrates how to use the SharePoint grounding tool with a Foundry agent| -|[Microsoft Fabric](./FoundryAgents_Step20_MicrosoftFabric/)|This sample demonstrates how to use Microsoft Fabric tool with a Foundry agent| -|[Web search](./FoundryAgents_Step21_WebSearch/)|This sample demonstrates how to use the Responses API web search tool with a Foundry agent| -|[Memory search](./FoundryAgents_Step22_MemorySearch/)|This sample demonstrates how to use memory search tool with a Foundry agent| -|[Local MCP](./FoundryAgents_Step23_LocalMCP/)|This sample demonstrates how to use a local MCP client with a Foundry agent| - -## Evaluation Samples - -Evaluation is critical for building trustworthy and high-quality AI applications. The evaluation samples demonstrate how to assess agent safety, quality, and performance using Azure AI Foundry's evaluation capabilities. - -|Sample|Description| -|---|---| -|[Red Team Evaluation](./FoundryAgents_Evaluations_Step01_RedTeaming/)|This sample demonstrates how to use Azure AI Foundry's Red Teaming service to assess model safety against adversarial attacks| -|[Self-Reflection with Groundedness](./FoundryAgents_Evaluations_Step02_SelfReflection/)|This sample demonstrates the self-reflection pattern where agents iteratively improve responses based on groundedness evaluation| - -For details on safety evaluation, see the [Red Team Evaluation README](./FoundryAgents_Evaluations_Step01_RedTeaming/README.md). - -## Running the samples from the console - -To run the samples, navigate to the desired sample directory, e.g. - -```powershell -cd FoundryAgents_Step01.2_Running -``` - -Set the following environment variables: - -```powershell -$env:AZURE_AI_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Azure Foundry resource endpoint -$env:AZURE_AI_MODEL_DEPLOYMENT_NAME="gpt-4o-mini" # Optional, defaults to gpt-4o-mini -``` - -If the variables are not set, you will be prompted for the values when running the samples. - -Execute the following command to build the sample: - -```powershell -dotnet build -``` - -Execute the following command to run the sample: - -```powershell -dotnet run --no-build -``` - -Or just build and run in one step: - -```powershell -dotnet run -``` - -## Running the samples from Visual Studio - -Open the solution in Visual Studio and set the desired sample project as the startup project. Then, run the project using the built-in debugger or by pressing `F5`. - -You will be prompted for any required environment variables if they are not already set. - diff --git a/dotnet/samples/02-agents/ModelContextProtocol/FoundryAgent_Hosted_MCP/Program.cs b/dotnet/samples/02-agents/ModelContextProtocol/FoundryAgent_Hosted_MCP/Program.cs index e91ed4d15a..bedf87b8c5 100644 --- a/dotnet/samples/02-agents/ModelContextProtocol/FoundryAgent_Hosted_MCP/Program.cs +++ b/dotnet/samples/02-agents/ModelContextProtocol/FoundryAgent_Hosted_MCP/Program.cs @@ -5,9 +5,11 @@ // The sample first shows how to use MCP tools with auto approval, and then how to set up a tool that requires approval before it can be invoked and how to approve such a tool. using Azure.AI.Projects; +using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; +using OpenAI.Responses; var endpoint = Environment.GetEnvironmentVariable("AZURE_AI_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("AZURE_AI_PROJECT_ENDPOINT is not set."); var model = Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME") ?? "gpt-4.1-mini"; @@ -23,26 +25,22 @@ var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCre // Create an MCP tool definition that the agent can use. // In this case we allow the tool to always be called without approval. -var mcpTool = new HostedMcpServerTool( - serverName: "microsoft_learn", - serverAddress: "https://learn.microsoft.com/api/mcp") -{ - AllowedTools = ["microsoft_docs_search"], - ApprovalMode = HostedMcpServerToolApprovalMode.NeverRequire -}; +var mcpTool = ResponseTool.CreateMcpTool( + serverLabel: "microsoft_learn", + serverUri: new Uri("https://learn.microsoft.com/api/mcp"), + toolCallApprovalPolicy: new McpToolCallApprovalPolicy(GlobalMcpToolCallApprovalPolicy.NeverRequireApproval)); // Create a server side agent with the mcp tool, and expose it as an AIAgent. -AIAgent agent = await aiProjectClient.CreateAIAgentAsync( - model: model, - options: new() - { - Name = "MicrosoftLearnAgent", - ChatOptions = new() +AgentVersion agentVersion = await aiProjectClient.Agents.CreateAgentVersionAsync( + "MicrosoftLearnAgent", + new AgentVersionCreationOptions( + new PromptAgentDefinition(model: model) { Instructions = "You answer questions by searching the Microsoft Learn content only.", - Tools = [mcpTool] - }, - }); + Tools = { mcpTool } + })); + +AIAgent agent = aiProjectClient.AsAIAgent(agentVersion); // You can then invoke the agent like any other AIAgent. AgentSession session = await agent.CreateSessionAsync(); @@ -56,26 +54,23 @@ aiProjectClient.Agents.DeleteAgent(agent.Name); // Create an MCP tool definition that the agent can use. // In this case we require approval before the tool can be called. -var mcpToolWithApproval = new HostedMcpServerTool( - serverName: "microsoft_learn", - serverAddress: "https://learn.microsoft.com/api/mcp") -{ - AllowedTools = ["microsoft_docs_search"], - ApprovalMode = HostedMcpServerToolApprovalMode.AlwaysRequire -}; +var mcpToolWithApproval = ResponseTool.CreateMcpTool( + serverLabel: "microsoft_learn", + serverUri: new Uri("https://learn.microsoft.com/api/mcp"), + allowedTools: new McpToolFilter() { ToolNames = { "microsoft_docs_search" } }, + toolCallApprovalPolicy: new McpToolCallApprovalPolicy(GlobalMcpToolCallApprovalPolicy.AlwaysRequireApproval)); // Create an agent with the MCP tool that requires approval. -AIAgent agentWithRequiredApproval = await aiProjectClient.CreateAIAgentAsync( - model: model, - options: new() - { - Name = "MicrosoftLearnAgentWithApproval", - ChatOptions = new() +AgentVersion agentVersionWithApproval = await aiProjectClient.Agents.CreateAgentVersionAsync( + "MicrosoftLearnAgentWithApproval", + new AgentVersionCreationOptions( + new PromptAgentDefinition(model: model) { Instructions = "You answer questions by searching the Microsoft Learn content only.", - Tools = [mcpToolWithApproval] - }, - }); + Tools = { mcpToolWithApproval } + })); + +AIAgent agentWithRequiredApproval = aiProjectClient.AsAIAgent(agentVersionWithApproval); // You can then invoke the agent like any other AIAgent. // For simplicity, we are assuming here that only mcp tool approvals are pending. diff --git a/dotnet/samples/02-agents/README.md b/dotnet/samples/02-agents/README.md index b901645f88..5ff0db416d 100644 --- a/dotnet/samples/02-agents/README.md +++ b/dotnet/samples/02-agents/README.md @@ -1,22 +1,21 @@ -# Getting started +# Getting started -The getting started samples demonstrate the fundamental concepts and functionalities -of the agent framework. +The getting started samples demonstrate the fundamental concepts and functionality of the agent framework. ## Samples -|Sample|Description| -|---|---| -|[Agents](./Agents/README.md)|Step by step instructions for getting started with agents| -|[Foundry Agents](./FoundryAgents/README.md)|Getting started with Azure Foundry Agents| -|[Agent Providers](./AgentProviders/README.md)|Getting started with creating agents using various providers| -|[Agents With Retrieval Augmented Generation (RAG)](./AgentWithRAG/README.md)|Adding Retrieval Augmented Generation (RAG) capabilities to your agents.| -|[Agents With Memory](./AgentWithMemory/README.md)|Adding Memory capabilities to your agents.| -|[Agent Open Telemetry](./AgentOpenTelemetry/README.md)|Getting started with OpenTelemetry for agents| -|[Agent With OpenAI exchange types](./AgentWithOpenAI/README.md)|Using OpenAI exchange types with agents| -|[Agent With Anthropic](./AgentWithAnthropic/README.md)|Getting started with agents using Anthropic Claude| -|[Model Context Protocol](./ModelContextProtocol/README.md)|Getting started with Model Context Protocol| -|[Agent Skills](./AgentSkills/README.md)|Getting started with Agent Skills| -|[Declarative Agents](./DeclarativeAgents)|Loading and executing AI agents from YAML configuration files| │ -|[AG-UI](./AGUI/README.md)|Getting started with AG-UI (Agent UI Protocol) servers and clients| │ -|[Dev UI](./DevUI/README.md)|Interactive web interface for testing and debugging AI agents during development| \ No newline at end of file +| Sample | Description | +| --- | --- | +| [Agents](./Agents/README.md) | Step-by-step instructions for getting started with agents | +| [Agents with Foundry](./AgentsWithFoundry/README.md) | Foundry agent samples using `FoundryAgent` and `AIProjectClient.AsAIAgent(...)` | +| [Agent Providers](./AgentProviders/README.md) | Getting started with creating agents using various providers | +| [Agents With Retrieval Augmented Generation (RAG)](./AgentWithRAG/README.md) | Adding Retrieval Augmented Generation (RAG) capabilities to your agents | +| [Agents With Memory](./AgentWithMemory/README.md) | Adding memory capabilities to your agents | +| [Agent Open Telemetry](./AgentOpenTelemetry/README.md) | Getting started with OpenTelemetry for agents | +| [Agent With OpenAI exchange types](./AgentWithOpenAI/README.md) | Using OpenAI exchange types with agents | +| [Agent With Anthropic](./AgentWithAnthropic/README.md) | Getting started with agents using Anthropic Claude | +| [Model Context Protocol](./ModelContextProtocol/README.md) | Getting started with Model Context Protocol | +| [Agent Skills](./AgentSkills/README.md) | Getting started with Agent Skills | +| [Declarative Agents](./DeclarativeAgents) | Loading and executing AI agents from YAML configuration files | +| [AG-UI](./AGUI/README.md) | Getting started with AG-UI (Agent UI Protocol) servers and clients | +| [Dev UI](./DevUI/README.md) | Interactive web interface for testing and debugging AI agents during development | diff --git a/dotnet/samples/03-workflows/Agents/FoundryAgent/Program.cs b/dotnet/samples/03-workflows/Agents/FoundryAgent/Program.cs index 589eca2bbc..5b1f59ac62 100644 --- a/dotnet/samples/03-workflows/Agents/FoundryAgent/Program.cs +++ b/dotnet/samples/03-workflows/Agents/FoundryAgent/Program.cs @@ -1,8 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. using Azure.AI.Projects; +using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; using Microsoft.Agents.AI.Workflows; using Microsoft.Extensions.AI; @@ -68,15 +70,19 @@ public static class Program /// The target language for translation /// The to create the agent with. /// The model to use for the agent - /// A ChatClientAgent configured for the specified language - private static async Task CreateTranslationAgentAsync( + /// A FoundryAgent configured for the specified language + private static async Task CreateTranslationAgentAsync( string targetLanguage, AIProjectClient aiProjectClient, string model) { - return await aiProjectClient.CreateAIAgentAsync( - name: $"{targetLanguage} Translator", - model: model, - instructions: $"You are a translation assistant that translates the provided text to {targetLanguage}."); + AgentVersion agentVersion = await aiProjectClient.Agents.CreateAgentVersionAsync( + $"{targetLanguage} Translator", + new AgentVersionCreationOptions( + new PromptAgentDefinition(model: model) + { + Instructions = $"You are a translation assistant that translates the provided text to {targetLanguage}.", + })); + return aiProjectClient.AsAIAgent(agentVersion); } } diff --git a/dotnet/samples/03-workflows/Declarative/HostedWorkflow/Program.cs b/dotnet/samples/03-workflows/Declarative/HostedWorkflow/Program.cs index 5936aaf82f..7852a8730f 100644 --- a/dotnet/samples/03-workflows/Declarative/HostedWorkflow/Program.cs +++ b/dotnet/samples/03-workflows/Declarative/HostedWorkflow/Program.cs @@ -8,6 +8,7 @@ using Azure.AI.Projects; using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; using Microsoft.Extensions.AI; using Microsoft.Extensions.Configuration; using Shared.Foundry; @@ -49,7 +50,7 @@ internal sealed class Program string workflowInput = GetWorkflowInput(args); - AIAgent agent = aiProjectClient.AsAIAgent(agentVersion); + FoundryAgent agent = aiProjectClient.AsAIAgent(agentVersion); AgentSession session = await agent.CreateSessionAsync(); diff --git a/dotnet/samples/03-workflows/README.md b/dotnet/samples/03-workflows/README.md index 2b8d375654..1ab52106ec 100644 --- a/dotnet/samples/03-workflows/README.md +++ b/dotnet/samples/03-workflows/README.md @@ -1,6 +1,6 @@ -# Workflow Getting Started Samples +# Workflow Getting Started Samples -The getting started with workflow samples demonstrate the fundamental concepts and functionalities of workflows in Agent Framework. +The workflow samples demonstrate the fundamental concepts and functionality of workflows in Agent Framework. ## Samples Overview @@ -20,15 +20,13 @@ Please begin with the [Start Here](./_StartHere) samples in order. These three s | [Mixed Workflow with Agents and Executors](./_StartHere/06_MixedWorkflowAgentsAndExecutors) | Shows how to mix agents and executors with adapter pattern for type conversion and protocol handling | | [Writer-Critic Workflow](./_StartHere/07_WriterCriticWorkflow) | Demonstrates iterative refinement with quality gates, max iteration safety, multiple message handlers, and conditional routing for feedback loops | -Once completed, please proceed to other samples listed below. - -> Note that you don't need to follow a strict order after the foundational samples. However, some samples build upon concepts from previous ones, so it's beneficial to be aware of the dependencies. +Once completed, please proceed to the other samples listed below. ### Agents | Sample | Concepts | |--------|----------| -| [Foundry Agents in Workflows](./Agents/FoundryAgent) | Demonstrates using Azure Foundry Agents within a workflow | +| [Foundry Agents in Workflows](./Agents/FoundryAgent) | Demonstrates using Azure Foundry agents in a workflow through `ChatClientAgent` | | [Custom Agent Executors](./Agents/CustomAgentExecutors) | Shows how to create a custom agent executor for more complex scenarios | | [Workflow as an Agent](./Agents/WorkflowAsAnAgent) | Illustrates how to encapsulate a workflow as an agent | | [Group Chat with Tool Approval](./Agents/GroupChatToolApproval) | Shows multi-agent group chat with tool approval requests and human-in-the-loop interaction | @@ -58,25 +56,3 @@ Once completed, please proceed to other samples listed below. | [Edge Conditions](./ConditionalEdges/01_EdgeCondition) | Introduces conditional edges for dynamic routing based on executor outputs | | [Switch-Case Routing](./ConditionalEdges/02_SwitchCase) | Extends conditional edges with switch-case routing for multiple paths | | [Multi-Selection Routing](./ConditionalEdges/03_MultiSelection) | Demonstrates multi-selection routing where one executor can trigger multiple downstream executors | - -> These 3 samples build upon each other. It's recommended to explore them in sequence to fully grasp the concepts. - -### Declarative Workflows - -| Sample | Concepts | -|--------|----------| -| [Declarative](./Declarative) | Demonstrates execution of declartive workflows. | - -### Checkpointing - -| Sample | Concepts | -|--------|----------| -| [Checkpoint and Resume](./Checkpoint/CheckpointAndResume) | Introduces checkpoints for saving and restoring workflow state for time travel purposes | -| [Checkpoint and Rehydrate](./Checkpoint/CheckpointAndRehydrate) | Demonstrates hydrating a new workflow instance from a saved checkpoint | -| [Checkpoint with Human-in-the-Loop](./Checkpoint/CheckpointWithHumanInTheLoop) | Combines checkpointing with human-in-the-loop interactions | - -### Human-in-the-Loop - -| Sample | Concepts | -|--------|----------| -| [Basic Human-in-the-Loop](./HumanInTheLoop/HumanInTheLoopBasic) | Introduces human-in-the-loop interaction using input ports and external requests | diff --git a/dotnet/samples/05-end-to-end/A2AClientServer/A2AServer/HostAgentFactory.cs b/dotnet/samples/05-end-to-end/A2AClientServer/A2AServer/HostAgentFactory.cs index 1149f9a293..13c01be156 100644 --- a/dotnet/samples/05-end-to-end/A2AClientServer/A2AServer/HostAgentFactory.cs +++ b/dotnet/samples/05-end-to-end/A2AClientServer/A2AServer/HostAgentFactory.cs @@ -2,6 +2,7 @@ using A2A; using Azure.AI.Projects; +using Azure.AI.Projects.Agents; using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; @@ -19,8 +20,8 @@ internal static class HostAgentFactory // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. var aiProjectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()); - AIAgent agent = await aiProjectClient - .GetAIAgentAsync(agentName, tools: tools); + AgentRecord agentRecord = await aiProjectClient.Agents.GetAgentAsync(agentName); + AIAgent agent = aiProjectClient.AsAIAgent(agentRecord, tools: tools); AgentCard agentCard = agentType.ToUpperInvariant() switch { diff --git a/dotnet/samples/AGENTS.md b/dotnet/samples/AGENTS.md index 1578b39a26..f515f531eb 100644 --- a/dotnet/samples/AGENTS.md +++ b/dotnet/samples/AGENTS.md @@ -28,7 +28,7 @@ dotnet/samples/ │ ├── AGUI/ # AG-UI protocol samples │ ├── DeclarativeAgents/ # Declarative agent definitions │ ├── DevUI/ # DevUI samples -│ ├── FoundryAgents/ # Azure AI Foundry agent samples +│ ├── AgentsWithFoundry/ # Azure AI Foundry samples (FoundryAgent + AsAIAgent extensions) │ └── ModelContextProtocol/ # MCP server/client patterns ├── 03-workflows/ # Workflow patterns │ ├── _StartHere/ # Introductory workflow samples diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/ChatResponseUpdateAGUIExtensions.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/ChatResponseUpdateAGUIExtensions.cs index f5fb103bd4..b11475ddfc 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/ChatResponseUpdateAGUIExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/ChatResponseUpdateAGUIExtensions.cs @@ -341,8 +341,17 @@ internal static class ChatResponseUpdateAGUIExtensions }; string? currentMessageId = null; + string? streamingMessageId = null; await foreach (var chatResponse in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) { + // Generate a fallback MessageId when the provider doesn't supply one. + // This ensures all AGUI events have a valid messageId regardless of agent type. + if (string.IsNullOrWhiteSpace(chatResponse.MessageId)) + { + streamingMessageId ??= Guid.NewGuid().ToString("N"); + chatResponse.MessageId = streamingMessageId; + } + if (chatResponse is { Contents.Count: > 0 } && chatResponse.Contents[0] is TextContent && !string.Equals(currentMessageId, chatResponse.MessageId, StringComparison.Ordinal)) diff --git a/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectChatClient.cs b/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectChatClient.cs index 32bb08674b..ec788233ed 100644 --- a/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectChatClient.cs +++ b/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectChatClient.cs @@ -44,7 +44,7 @@ internal sealed class AzureAIProjectChatClient : DelegatingChatClient { this._agentClient = aiProjectClient; this._agentReference = Throw.IfNull(agentReference); - this._metadata = new ChatClientMetadata("azure.ai.agents", defaultModelId: defaultModelId); + this._metadata = new ChatClientMetadata("microsoft.foundry", defaultModelId: defaultModelId); this._chatOptions = chatOptions; } diff --git a/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectChatClientExtensions.cs b/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectChatClientExtensions.cs index b129f4b1f2..479ab894ae 100644 --- a/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectChatClientExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectChatClientExtensions.cs @@ -14,6 +14,7 @@ using Azure.AI.Projects.Agents; using Microsoft.Agents.AI; using Microsoft.Agents.AI.AzureAI; using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; using OpenAI; @@ -42,7 +43,7 @@ public static partial class AzureAIProjectChatClientExtensions /// When instantiating a by using an , minimal information will be available about the agent in the instance level, and any logic that relies /// on to retrieve information about the agent like will receive as the result. /// - public static ChatClientAgent AsAIAgent( + public static FoundryAgent AsAIAgent( this AIProjectClient aiProjectClient, AgentReference agentReference, IList? tools = null, @@ -53,7 +54,7 @@ public static partial class AzureAIProjectChatClientExtensions Throw.IfNull(agentReference); ThrowIfInvalidAgentName(agentReference.Name); - return AsChatClientAgent( + var innerAgent = AsChatClientAgent( aiProjectClient, agentReference, new ChatClientAgentOptions() @@ -64,6 +65,8 @@ public static partial class AzureAIProjectChatClientExtensions }, clientFactory, services); + + return new FoundryAgent(aiProjectClient, innerAgent); } /// @@ -79,7 +82,8 @@ public static partial class AzureAIProjectChatClientExtensions /// Thrown when or is . /// Thrown when is empty or whitespace, or when the agent with the specified name was not found. /// The agent with the specified name was not found. - public static async Task GetAIAgentAsync( + [Obsolete("Use native AIProjectClient agent APIs and AsAIAgent(AgentRecord/AgentVersion) instead.")] + public static async Task GetAIAgentAsync( this AIProjectClient aiProjectClient, string name, IList? tools = null, @@ -110,7 +114,7 @@ public static partial class AzureAIProjectChatClientExtensions /// An optional to use for resolving services required by the instances being invoked. /// A instance that can be used to perform operations based on the latest version of the Azure AI Agent. /// Thrown when or is . - public static ChatClientAgent AsAIAgent( + public static FoundryAgent AsAIAgent( this AIProjectClient aiProjectClient, AgentRecord agentRecord, IList? tools = null, @@ -122,13 +126,15 @@ public static partial class AzureAIProjectChatClientExtensions var allowDeclarativeMode = tools is not { Count: > 0 }; - return AsChatClientAgent( + var innerAgent = AsChatClientAgent( aiProjectClient, agentRecord, tools, clientFactory, !allowDeclarativeMode, services); + + return new FoundryAgent(aiProjectClient, innerAgent); } /// @@ -141,7 +147,7 @@ public static partial class AzureAIProjectChatClientExtensions /// An optional to use for resolving services required by the instances being invoked. /// A instance that can be used to perform operations based on the provided version of the Azure AI Agent. /// Thrown when or is . - public static ChatClientAgent AsAIAgent( + public static FoundryAgent AsAIAgent( this AIProjectClient aiProjectClient, AgentVersion agentVersion, IList? tools = null, @@ -153,13 +159,15 @@ public static partial class AzureAIProjectChatClientExtensions var allowDeclarativeMode = tools is not { Count: > 0 }; - return AsChatClientAgent( + var innerAgent = AsChatClientAgent( aiProjectClient, agentVersion, tools, clientFactory, !allowDeclarativeMode, services); + + return new FoundryAgent(aiProjectClient, innerAgent); } /// @@ -172,7 +180,8 @@ public static partial class AzureAIProjectChatClientExtensions /// A to cancel the operation if needed. /// A instance that can be used to perform operations on the newly created agent. /// Thrown when or is . - public static async Task GetAIAgentAsync( + [Obsolete("Use native AIProjectClient agent APIs and AsAIAgent(AgentRecord/AgentVersion) instead.")] + public static async Task GetAIAgentAsync( this AIProjectClient aiProjectClient, ChatClientAgentOptions options, Func? clientFactory = null, @@ -194,12 +203,9 @@ public static partial class AzureAIProjectChatClientExtensions var agentOptions = CreateChatClientAgentOptions(agentVersion, options, requireInvocableTools: !options.UseProvidedChatClientAsIs); - return AsChatClientAgent( + return new FoundryAgent( aiProjectClient, - agentVersion, - agentOptions, - clientFactory, - services); + AsChatClientAgent(aiProjectClient, agentVersion, agentOptions, clientFactory, services)); } /// @@ -218,7 +224,8 @@ public static partial class AzureAIProjectChatClientExtensions /// Thrown when , , or is . /// Thrown when or is empty or whitespace. /// When using prompt agent definitions with tools the parameter needs to be provided. - public static Task CreateAIAgentAsync( + [Obsolete("Use native AIProjectClient.Agents APIs instead.")] + public static Task CreateAIAgentAsync( this AIProjectClient aiProjectClient, string name, string model, @@ -256,7 +263,8 @@ public static partial class AzureAIProjectChatClientExtensions /// A instance that can be used to perform operations on the newly created agent. /// Thrown when or is . /// Thrown when is empty or whitespace, or when the agent name is not provided in the options. - public static async Task CreateAIAgentAsync( + [Obsolete("Use native AIProjectClient.Agents APIs instead.")] + public static async Task CreateAIAgentAsync( this AIProjectClient aiProjectClient, string model, ChatClientAgentOptions options, @@ -267,7 +275,6 @@ public static partial class AzureAIProjectChatClientExtensions Throw.IfNull(aiProjectClient); Throw.IfNull(options); Throw.IfNullOrWhitespace(model); - const bool RequireInvocableTools = true; if (string.IsNullOrWhiteSpace(options.Name)) { @@ -276,43 +283,13 @@ public static partial class AzureAIProjectChatClientExtensions ThrowIfInvalidAgentName(options.Name); - PromptAgentDefinition agentDefinition = new(model) - { - Instructions = options.ChatOptions?.Instructions, - Temperature = options.ChatOptions?.Temperature, - TopP = options.ChatOptions?.TopP, - TextOptions = new() { TextFormat = ToOpenAIResponseTextFormat(options.ChatOptions?.ResponseFormat, options.ChatOptions) } - }; + AgentVersion agentVersion = await CreateAgentVersionFromOptionsAsync(aiProjectClient, model, options, cancellationToken).ConfigureAwait(false); - // Map reasoning options from the abstraction-level ChatOptions.Reasoning, - // falling back to extracting from the raw representation factory for breaking glass scenarios. - if (options.ChatOptions?.Reasoning is { } reasoning) - { - agentDefinition.ReasoningOptions = ToResponseReasoningOptions(reasoning); - } - else if (options.ChatOptions?.RawRepresentationFactory?.Invoke(new NoOpChatClient()) is CreateResponseOptions respCreationOptions) - { - agentDefinition.ReasoningOptions = respCreationOptions.ReasoningOptions; - } + var agentOptions = CreateChatClientAgentOptions(agentVersion, options, requireInvocableTools: true); - ApplyToolsToAgentDefinition(agentDefinition, options.ChatOptions?.Tools); - - AgentVersionCreationOptions? creationOptions = new(agentDefinition); - if (!string.IsNullOrWhiteSpace(options.Description)) - { - creationOptions.Description = options.Description; - } - - AgentVersion agentVersion = await CreateAgentVersionWithProtocolAsync(aiProjectClient, options.Name, creationOptions, cancellationToken).ConfigureAwait(false); - - var agentOptions = CreateChatClientAgentOptions(agentVersion, options, RequireInvocableTools); - - return AsChatClientAgent( + return new FoundryAgent( aiProjectClient, - agentVersion, - agentOptions, - clientFactory, - services); + AsChatClientAgent(aiProjectClient, agentVersion, agentOptions, clientFactory, services)); } /// @@ -330,7 +307,8 @@ public static partial class AzureAIProjectChatClientExtensions /// When using this extension method with a the tools are only declarative and not invocable. /// Invocation of any in-process tools will need to be handled manually. /// - public static Task CreateAIAgentAsync( + [Obsolete("Use native AIProjectClient.Agents APIs instead.")] + public static Task CreateAIAgentAsync( this AIProjectClient aiProjectClient, string name, AgentVersionCreationOptions creationOptions, @@ -351,6 +329,75 @@ public static partial class AzureAIProjectChatClientExtensions cancellationToken); } + /// + /// Creates a non-versioned backed by the project's Responses API using the specified model and instructions. + /// + /// The to use for Responses API calls. Cannot be . + /// The model deployment name to use for the agent. Cannot be or whitespace. + /// The instructions that guide the agent's behavior. Cannot be or whitespace. + /// Optional name for the agent. + /// Optional human-readable description for the agent. + /// Optional collection of tools that the agent can invoke during conversations. + /// Provides a way to customize the creation of the underlying used by the agent. + /// Optional logger factory for creating loggers used by the agent. + /// An optional to use for resolving services required by the instances being invoked. + /// A backed by the project's Responses API. + /// Thrown when is . + /// Thrown when or is empty or whitespace. + public static FoundryAgent AsAIAgent( + this AIProjectClient aiProjectClient, + string model, + string instructions, + string? name = null, + string? description = null, + IList? tools = null, + Func? clientFactory = null, + ILoggerFactory? loggerFactory = null, + IServiceProvider? services = null) + { + Throw.IfNull(aiProjectClient); + Throw.IfNullOrWhitespace(model); + Throw.IfNullOrWhitespace(instructions); + + ChatClientAgentOptions options = new() + { + Name = name, + Description = description, + ChatOptions = new ChatOptions + { + ModelId = model, + Instructions = instructions, + Tools = tools, + }, + }; + + return new FoundryAgent(aiProjectClient, CreateResponsesChatClientAgent(aiProjectClient, options, clientFactory, loggerFactory, services)); + } + + /// + /// Creates a non-versioned backed by the project's Responses API using the specified options. + /// + /// The to use for Responses API calls. Cannot be . + /// Configuration options that control the agent's behavior. is required. + /// Provides a way to customize the creation of the underlying used by the agent. + /// Optional logger factory for creating loggers used by the agent. + /// An optional to use for resolving services required by the instances being invoked. + /// A backed by the project's Responses API. + /// Thrown when or is . + /// Thrown when does not specify . + public static FoundryAgent AsAIAgent( + this AIProjectClient aiProjectClient, + ChatClientAgentOptions options, + Func? clientFactory = null, + ILoggerFactory? loggerFactory = null, + IServiceProvider? services = null) + { + Throw.IfNull(aiProjectClient); + Throw.IfNull(options); + + return new FoundryAgent(aiProjectClient, CreateResponsesChatClientAgent(aiProjectClient, options, clientFactory, loggerFactory, services)); + } + #region Private private static readonly ModelReaderWriterOptions s_modelWriterOptionsWire = new("W"); @@ -358,7 +405,7 @@ public static partial class AzureAIProjectChatClientExtensions /// /// Asynchronously retrieves an agent record by name using the protocol method to inject user-agent headers. /// - private static async Task GetAgentRecordByNameAsync(AIProjectClient aiProjectClient, string agentName, CancellationToken cancellationToken) + internal static async Task GetAgentRecordByNameAsync(AIProjectClient aiProjectClient, string agentName, CancellationToken cancellationToken) { ClientResult protocolResponse = await aiProjectClient.Agents.GetAgentAsync(agentName, cancellationToken.ToRequestOptions(false)).ConfigureAwait(false); var rawResponse = protocolResponse.GetRawResponse(); @@ -369,7 +416,7 @@ public static partial class AzureAIProjectChatClientExtensions /// /// Asynchronously creates an agent version using the protocol method to inject user-agent headers. /// - private static async Task CreateAgentVersionWithProtocolAsync(AIProjectClient aiProjectClient, string agentName, AgentVersionCreationOptions creationOptions, CancellationToken cancellationToken) + internal static async Task CreateAgentVersionWithProtocolAsync(AIProjectClient aiProjectClient, string agentName, AgentVersionCreationOptions creationOptions, CancellationToken cancellationToken) { BinaryData serializedOptions = ModelReaderWriter.Write(creationOptions, s_modelWriterOptionsWire, AzureAIProjectsAgentsContext.Default); BinaryContent content = BinaryContent.Create(serializedOptions); @@ -379,7 +426,7 @@ public static partial class AzureAIProjectChatClientExtensions return result ?? throw new InvalidOperationException($"Failed to create agent version for agent '{agentName}'."); } - private static async Task CreateAIAgentAsync( + private static async Task CreateAIAgentAsync( this AIProjectClient aiProjectClient, string name, IList? tools, @@ -397,17 +444,61 @@ public static partial class AzureAIProjectChatClientExtensions AgentVersion agentVersion = await CreateAgentVersionWithProtocolAsync(aiProjectClient, name, creationOptions, cancellationToken).ConfigureAwait(false); - return AsChatClientAgent( - aiProjectClient, - agentVersion, - tools, - clientFactory, - !allowDeclarativeMode, - services); + return new FoundryAgent(aiProjectClient, AsChatClientAgent(aiProjectClient, agentVersion, tools, clientFactory, !allowDeclarativeMode, services)); } - /// This method creates an with the specified ChatClientAgentOptions. - private static ChatClientAgent AsChatClientAgent( + /// + /// Creates an agent version with optional tool application, using the protocol method to inject user-agent headers. + /// + internal static async Task CreateAgentVersionWithProtocolAsync(AIProjectClient aiProjectClient, string agentName, AgentVersionCreationOptions creationOptions, IList? tools, CancellationToken cancellationToken) + { + if (tools is { Count: > 0 }) + { + ApplyToolsToAgentDefinition(creationOptions.Definition, tools); + } + + return await CreateAgentVersionWithProtocolAsync(aiProjectClient, agentName, creationOptions, cancellationToken).ConfigureAwait(false); + } + + /// + /// Creates an agent version from , mapping options to a . + /// + internal static async Task CreateAgentVersionFromOptionsAsync( + AIProjectClient aiProjectClient, + string model, + ChatClientAgentOptions options, + CancellationToken cancellationToken) + { + PromptAgentDefinition agentDefinition = new(model) + { + Instructions = options.ChatOptions?.Instructions, + Temperature = options.ChatOptions?.Temperature, + TopP = options.ChatOptions?.TopP, + TextOptions = new() { TextFormat = ToOpenAIResponseTextFormat(options.ChatOptions?.ResponseFormat, options.ChatOptions) } + }; + + if (options.ChatOptions?.Reasoning is { } reasoning) + { + agentDefinition.ReasoningOptions = ToResponseReasoningOptions(reasoning); + } + else if (options.ChatOptions?.RawRepresentationFactory?.Invoke(new NoOpChatClient()) is CreateResponseOptions respCreationOptions) + { + agentDefinition.ReasoningOptions = respCreationOptions.ReasoningOptions; + } + + ApplyToolsToAgentDefinition(agentDefinition, options.ChatOptions?.Tools); + + AgentVersionCreationOptions creationOptions = new(agentDefinition); + if (!string.IsNullOrWhiteSpace(options.Description)) + { + creationOptions.Description = options.Description; + } + + return await CreateAgentVersionWithProtocolAsync(aiProjectClient, options.Name!, creationOptions, cancellationToken).ConfigureAwait(false); + } + + /// Creates a with the specified options. + internal static ChatClientAgent CreateChatClientAgent( AIProjectClient aiProjectClient, AgentVersion agentVersion, ChatClientAgentOptions agentOptions, @@ -424,6 +515,37 @@ public static partial class AzureAIProjectChatClientExtensions return new ChatClientAgent(chatClient, agentOptions, services: services); } + internal static ChatClientAgent CreateResponsesChatClientAgent( + AIProjectClient aiProjectClient, + ChatClientAgentOptions agentOptions, + Func? clientFactory, + ILoggerFactory? loggerFactory, + IServiceProvider? services) + { + Throw.IfNull(aiProjectClient); + Throw.IfNull(agentOptions); + Throw.IfNull(agentOptions.ChatOptions); + Throw.IfNullOrWhitespace(agentOptions.ChatOptions.ModelId); + + IChatClient chatClient = new AzureAIProjectResponsesChatClient(aiProjectClient, agentOptions.ChatOptions.ModelId); + + if (clientFactory is not null) + { + chatClient = clientFactory(chatClient); + } + + return new ChatClientAgent(chatClient, agentOptions, loggerFactory, services); + } + + /// This method creates an with the specified ChatClientAgentOptions. + private static ChatClientAgent AsChatClientAgent( + AIProjectClient aiProjectClient, + AgentVersion agentVersion, + ChatClientAgentOptions agentOptions, + Func? clientFactory, + IServiceProvider? services) + => CreateChatClientAgent(aiProjectClient, agentVersion, agentOptions, clientFactory, services); + /// This method creates an with the specified ChatClientAgentOptions. private static ChatClientAgent AsChatClientAgent( AIProjectClient aiProjectClient, @@ -503,7 +625,7 @@ public static partial class AzureAIProjectChatClientExtensions /// This method rebuilds the agent options from the agent definition returned by the version and combine with the in-proc tools when provided /// this ensures that all required tools are provided and the definition of the agent options are consistent with the agent definition coming from the server. /// - private static ChatClientAgentOptions CreateChatClientAgentOptions(AgentVersion agentVersion, ChatOptions? chatOptions, bool requireInvocableTools) + internal static ChatClientAgentOptions CreateChatClientAgentOptions(AgentVersion agentVersion, ChatOptions? chatOptions, bool requireInvocableTools) { var agentDefinition = agentVersion.Definition; @@ -591,7 +713,7 @@ public static partial class AzureAIProjectChatClientExtensions /// Specifies whether the returned options must include invocable tools. Set to to require /// invocable tools; otherwise, . /// A instance configured according to the specified parameters. - private static ChatClientAgentOptions CreateChatClientAgentOptions(AgentVersion agentVersion, ChatClientAgentOptions? options, bool requireInvocableTools) + internal static ChatClientAgentOptions CreateChatClientAgentOptions(AgentVersion agentVersion, ChatClientAgentOptions? options, bool requireInvocableTools) { var agentOptions = CreateChatClientAgentOptions(agentVersion, options?.ChatOptions, requireInvocableTools); if (options is not null) @@ -768,7 +890,7 @@ public static partial class AzureAIProjectChatClientExtensions private static Regex AgentNameValidationRegex() => new("^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$"); #endif - private static string ThrowIfInvalidAgentName(string? name) + internal static string ThrowIfInvalidAgentName(string? name) { Throw.IfNullOrWhitespace(name); if (!AgentNameValidationRegex().IsMatch(name)) diff --git a/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectResponsesChatClient.cs b/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectResponsesChatClient.cs new file mode 100644 index 0000000000..48bb20d766 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AzureAI/AzureAIProjectResponsesChatClient.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Azure.AI.Projects; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.AzureAI; + +#pragma warning disable OPENAI001 +internal sealed class AzureAIProjectResponsesChatClient : DelegatingChatClient +{ + private readonly ChatClientMetadata _metadata; + private readonly AIProjectClient _aiProjectClient; + + internal AzureAIProjectResponsesChatClient(AIProjectClient aiProjectClient, string defaultModelId) + : base(Throw.IfNull(aiProjectClient) + .GetProjectOpenAIClient() + .GetProjectResponsesClientForModel(Throw.IfNullOrWhitespace(defaultModelId)) + .AsIChatClient()) + { + this._aiProjectClient = aiProjectClient; + this._metadata = new ChatClientMetadata("microsoft.foundry", defaultModelId: defaultModelId); + } + + public override object? GetService(Type serviceType, object? serviceKey = null) + { + return (serviceKey is null && serviceType == typeof(ChatClientMetadata)) + ? this._metadata + : (serviceKey is null && serviceType == typeof(AIProjectClient)) + ? this._aiProjectClient + : base.GetService(serviceType, serviceKey); + } +} +#pragma warning restore OPENAI001 diff --git a/dotnet/src/Microsoft.Agents.AI.AzureAI/CompatibilitySuppressions.xml b/dotnet/src/Microsoft.Agents.AI.AzureAI/CompatibilitySuppressions.xml new file mode 100644 index 0000000000..79cd4b890d --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AzureAI/CompatibilitySuppressions.xml @@ -0,0 +1,284 @@ + + + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentRecord,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentReference,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentVersion,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Azure.AI.Projects.AgentVersionCreationOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.Threading.CancellationToken) + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentRecord,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net472/Microsoft.Agents.AI.AzureAI.dll + lib/net472/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentReference,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net472/Microsoft.Agents.AI.AzureAI.dll + lib/net472/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentVersion,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net472/Microsoft.Agents.AI.AzureAI.dll + lib/net472/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Azure.AI.Projects.AgentVersionCreationOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.Threading.CancellationToken) + lib/net472/Microsoft.Agents.AI.AzureAI.dll + lib/net472/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net472/Microsoft.Agents.AI.AzureAI.dll + lib/net472/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net472/Microsoft.Agents.AI.AzureAI.dll + lib/net472/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net472/Microsoft.Agents.AI.AzureAI.dll + lib/net472/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net472/Microsoft.Agents.AI.AzureAI.dll + lib/net472/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentRecord,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentReference,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentVersion,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Azure.AI.Projects.AgentVersionCreationOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.Threading.CancellationToken) + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentRecord,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentReference,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentVersion,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Azure.AI.Projects.AgentVersionCreationOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.Threading.CancellationToken) + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentRecord,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentReference,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.AsAIAgent(Azure.AI.Projects.AIProjectClient,Azure.AI.Projects.OpenAI.AgentVersion,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider) + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Azure.AI.Projects.AgentVersionCreationOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.Threading.CancellationToken) + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.CreateAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + true + + + CP0002 + M:Azure.AI.Projects.AzureAIProjectChatClientExtensions.GetAIAgentAsync(Azure.AI.Projects.AIProjectClient,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},System.IServiceProvider,System.Threading.CancellationToken) + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.AzureAI.dll + true + + \ No newline at end of file diff --git a/dotnet/src/Microsoft.Agents.AI.AzureAI/FoundryAITool.cs b/dotnet/src/Microsoft.Agents.AI.AzureAI/FoundryAITool.cs new file mode 100644 index 0000000000..80ed48e1df --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AzureAI/FoundryAITool.cs @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Azure.AI.Projects.Agents; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using OpenAI.Responses; + +#pragma warning disable OPENAI001 + +namespace Microsoft.Agents.AI.AzureAI; + +/// +/// Provides factory methods for creating instances from Microsoft Foundry and OpenAI response tools. +/// +/// +/// +/// This class wraps (Azure.AI.Projects.OpenAI) and (OpenAI SDK) factory methods, +/// returning directly — eliminating the need for manual casting and .AsAITool() calls. +/// +/// +/// Instead of writing: +/// ((ResponseTool)AgentTool.CreateOpenApiTool(definition)).AsAITool() +/// You can write: +/// FoundryAITool.CreateOpenApiTool(definition) +/// +/// +[Experimental(DiagnosticIds.Experiments.AIOpenAIResponses)] +public static class FoundryAITool +{ + /// + /// Converts an existing into an . + /// + /// The response tool to convert. + /// An wrapping the provided response tool. + public static AITool FromResponseTool(ResponseTool responseTool) => responseTool.AsAITool(); + + // --- Azure.AI.Projects.OpenAI AgentTool factories --- + + /// + /// Creates an for OpenAPI tool invocations. + /// + /// The OpenAPI function definition specifying the API endpoint, schema, and authentication. + /// An that calls the specified OpenAPI endpoint. + public static AITool CreateOpenApiTool(OpenApiFunctionDefinition definition) + => ((ResponseTool)AgentTool.CreateOpenApiTool(definition)).AsAITool(); + + /// + /// Creates an for Bing Grounding search. + /// + /// The Bing Grounding search configuration options. + /// An for Bing Grounding search. + public static AITool CreateBingGroundingTool(BingGroundingSearchToolOptions options) + => ((ResponseTool)AgentTool.CreateBingGroundingTool(options)).AsAITool(); + + /// + /// Creates an for Bing Custom Search. + /// + /// The Bing Custom Search configuration parameters. + /// An for Bing Custom Search. + public static AITool CreateBingCustomSearchTool(BingCustomSearchToolOptions parameters) + => ((ResponseTool)AgentTool.CreateBingCustomSearchTool(parameters)).AsAITool(); + + /// + /// Creates an for Microsoft Fabric data agent. + /// + /// The Fabric data agent configuration options. + /// An for Microsoft Fabric. + public static AITool CreateMicrosoftFabricTool(FabricDataAgentToolOptions options) + => ((ResponseTool)AgentTool.CreateMicrosoftFabricTool(options)).AsAITool(); + + /// + /// Creates an for SharePoint grounding. + /// + /// The SharePoint grounding configuration options. + /// An for SharePoint grounding. + public static AITool CreateSharepointTool(SharePointGroundingToolOptions options) + => ((ResponseTool)AgentTool.CreateSharepointTool(options)).AsAITool(); + + /// + /// Creates an for Azure AI Search. + /// + /// Optional Azure AI Search configuration options. + /// An for Azure AI Search. + public static AITool CreateAzureAISearchTool(AzureAISearchToolOptions? options = null) + => ((ResponseTool)AgentTool.CreateAzureAISearchTool(options)).AsAITool(); + + /// + /// Creates an for browser automation. + /// + /// The browser automation configuration parameters. + /// An for browser automation. + public static AITool CreateBrowserAutomationTool(BrowserAutomationToolOptions parameters) + => ((ResponseTool)AgentTool.CreateBrowserAutomationTool(parameters)).AsAITool(); + + /// + /// Creates an for structured output capture. + /// + /// The structured output definition. + /// An for structured output capture. + public static AITool CreateStructuredOutputsTool(StructuredOutputDefinition outputs) + => ((ResponseTool)AgentTool.CreateStructuredOutputsTool(outputs)).AsAITool(); + + /// + /// Creates an for Agent-to-Agent (A2A) communication. + /// + /// The base URI for the A2A agent. + /// Optional path to the agent card. + /// An for A2A communication. + public static AITool CreateA2ATool(Uri baseUri, string? agentCardPath = null) + => AgentTool.CreateA2ATool(baseUri, agentCardPath).AsAITool(); + + // --- OpenAI SDK ResponseTool factories --- + + /// + /// Creates an for computer use (screen interaction). + /// + /// The computer tool environment type. + /// The display width in pixels. + /// The display height in pixels. + /// An for computer use. + [Experimental("OPENAICUA001")] + public static AITool CreateComputerTool(ComputerToolEnvironment environment, int displayWidth, int displayHeight) + => ResponseTool.CreateComputerTool(environment, displayWidth, displayHeight).AsAITool(); + + /// + /// Creates an for function tool invocations. + /// + /// The name of the function. + /// The function parameters schema as JSON. + /// Whether strict mode is enabled for parameter validation. + /// Optional description of the function. + /// An for function invocations. + public static AITool CreateFunctionTool(string functionName, BinaryData functionParameters, bool? strictModeEnabled, string? functionDescription = null) + => ResponseTool.CreateFunctionTool(functionName, functionParameters, strictModeEnabled, functionDescription).AsAITool(); + + /// + /// Creates an for file search over vector stores. + /// + /// The IDs of vector stores to search. + /// Optional maximum number of results to return. + /// Optional ranking options for search results. + /// Optional filters for search results. + /// An for file search. + public static AITool CreateFileSearchTool(IEnumerable vectorStoreIds, int? maxResultCount = null, FileSearchToolRankingOptions? rankingOptions = null, BinaryData? filters = null) + => ResponseTool.CreateFileSearchTool(vectorStoreIds, maxResultCount, rankingOptions, filters).AsAITool(); + + /// + /// Creates an for web search. + /// + /// Optional user location for search context. + /// Optional search context size. + /// Optional search filters. + /// An for web search. + public static AITool CreateWebSearchTool(WebSearchToolLocation? userLocation = null, WebSearchToolContextSize? searchContextSize = null, WebSearchToolFilters? filters = null) + => ResponseTool.CreateWebSearchTool(userLocation, searchContextSize, filters).AsAITool(); + + /// + /// Creates an for MCP (Model Context Protocol) server tools. + /// + /// The label for the MCP server. + /// The URI of the MCP server. + /// Optional authorization token. + /// Optional server description. + /// Optional custom headers. + /// Optional filter for allowed tools. + /// Optional tool call approval policy. + /// An for MCP server tools. + public static AITool CreateMcpTool(string serverLabel, Uri serverUri, string? authorizationToken = null, string? serverDescription = null, IDictionary? headers = null, McpToolFilter? allowedTools = null, McpToolCallApprovalPolicy? toolCallApprovalPolicy = null) + => ResponseTool.CreateMcpTool(serverLabel, serverUri, authorizationToken, serverDescription, headers, allowedTools, toolCallApprovalPolicy).AsAITool(); + + /// + /// Creates an for MCP (Model Context Protocol) server tools using a connector ID. + /// + /// The label for the MCP server. + /// The connector ID for the MCP server. + /// Optional authorization token. + /// Optional server description. + /// Optional custom headers. + /// Optional filter for allowed tools. + /// Optional tool call approval policy. + /// An for MCP server tools. + public static AITool CreateMcpTool(string serverLabel, McpToolConnectorId connectorId, string? authorizationToken = null, string? serverDescription = null, IDictionary? headers = null, McpToolFilter? allowedTools = null, McpToolCallApprovalPolicy? toolCallApprovalPolicy = null) + => ResponseTool.CreateMcpTool(serverLabel, connectorId, authorizationToken, serverDescription, headers, allowedTools, toolCallApprovalPolicy).AsAITool(); + + /// + /// Creates an for code interpreter. + /// + /// The container configuration for the code interpreter. + /// An for code interpreter. + public static AITool CreateCodeInterpreterTool(CodeInterpreterToolContainer container) + => ResponseTool.CreateCodeInterpreterTool(container).AsAITool(); + + /// + /// Creates an for image generation. + /// + /// The model to use for image generation. + /// Optional image quality setting. + /// Optional image size setting. + /// Optional output file format. + /// Optional output compression factor. + /// Optional moderation level. + /// Optional background setting. + /// Optional input fidelity setting. + /// Optional input image mask. + /// Optional partial image count. + /// An for image generation. + public static AITool CreateImageGenerationTool(string model, ImageGenerationToolQuality? quality = null, ImageGenerationToolSize? size = null, ImageGenerationToolOutputFileFormat? outputFileFormat = null, int? outputCompressionFactor = null, ImageGenerationToolModerationLevel? moderationLevel = null, ImageGenerationToolBackground? background = null, ImageGenerationToolInputFidelity? inputFidelity = null, ImageGenerationToolInputImageMask? inputImageMask = null, int? partialImageCount = null) + => ResponseTool.CreateImageGenerationTool(model, quality, size, outputFileFormat, outputCompressionFactor, moderationLevel, background, inputFidelity, inputImageMask, partialImageCount).AsAITool(); +} diff --git a/dotnet/src/Microsoft.Agents.AI.AzureAI/FoundryAgent.cs b/dotnet/src/Microsoft.Agents.AI.AzureAI/FoundryAgent.cs new file mode 100644 index 0000000000..66fa93e39f --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AzureAI/FoundryAgent.cs @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ClientModel; +using System.Diagnostics.CodeAnalysis; +using Azure.AI.Extensions.OpenAI; +using Azure.AI.Projects; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.AzureAI; + +/// +/// Provides an that uses Microsoft Foundry for AI agent capabilities. +/// +/// +/// +/// connects to a pre-configured server-side agent in Microsoft Foundry, +/// wrapping it as an for use with Agent Framework. Unlike the direct +/// AIProjectClient.AsAIAgent(model, instructions) approach (which creates a local agent +/// backed by the Responses API without any server-side agent definition), +/// works with agents that are managed and versioned in the Foundry service. +/// +/// +/// This class provides convenient access to Foundry-specific features such as server-side +/// conversation management via . +/// +/// +/// Instances can be created directly via public constructors or through +/// AsAIAgent extension methods on . +/// +/// +[Experimental(DiagnosticIds.Experiments.AIOpenAIResponses)] +public sealed class FoundryAgent : DelegatingAIAgent +{ + private readonly AIProjectClient _aiProjectClient; + + /// + /// Initializes a new instance of the class using the direct Responses API path. + /// + /// The Microsoft Foundry project endpoint. + /// The authentication credential. + /// The model deployment name. + /// The instructions that guide the agent's behavior. + /// Optional configuration options for the . + /// Optional name for the agent. + /// Optional description for the agent. + /// Optional tools to use when interacting with the agent. + /// Provides a way to customize the creation of the underlying . + /// Optional logger factory for creating loggers used by the agent. + /// Optional service provider for resolving dependencies required by AI functions. + public FoundryAgent( + Uri projectEndpoint, + AuthenticationTokenProvider credential, + string model, + string instructions, + AIProjectClientOptions? clientOptions = null, + string? name = null, + string? description = null, + IList? tools = null, + Func? clientFactory = null, + ILoggerFactory? loggerFactory = null, + IServiceProvider? services = null) + : base(CreateInnerAgent( + CreateProjectClient(projectEndpoint, credential, clientOptions), + model, instructions, name, description, tools, clientFactory, loggerFactory, services, + out var aiProjectClient)) + { + this._aiProjectClient = aiProjectClient; + } + + /// + /// Initializes a new instance of the class from an agent-specific endpoint. + /// + /// The agent-specific endpoint URI (must contain the agent name in the path). + /// The authentication credential. + /// Optional configuration options for the . + /// Optional tools to use when interacting with the agent. + /// Provides a way to customize the creation of the underlying . + /// Optional service provider for resolving dependencies required by AI functions. + public FoundryAgent( + Uri agentEndpoint, + AuthenticationTokenProvider credential, + AIProjectClientOptions? clientOptions = null, + IList? tools = null, + Func? clientFactory = null, + IServiceProvider? services = null) + : base(CreateInnerAgentFromEndpoint( + CreateProjectClient(agentEndpoint, credential, clientOptions), + agentEndpoint, tools, clientFactory, services, + out var aiProjectClient)) + { + this._aiProjectClient = aiProjectClient; + } + + /// + /// Internal constructor used by AsAIAgent extension methods that already have an and a configured . + /// + internal FoundryAgent(AIProjectClient aiProjectClient, ChatClientAgent innerAgent) + : base(Throw.IfNull(innerAgent)) + { + this._aiProjectClient = Throw.IfNull(aiProjectClient); + } + + #region Convenience methods + + /// + /// Creates a server-side conversation session that appears in the Foundry Project UI. + /// + /// A token to monitor for cancellation requests. + /// A linked to the newly created server-side conversation. + public async Task CreateConversationSessionAsync(CancellationToken cancellationToken = default) + { + var conversationsClient = this._aiProjectClient + .GetProjectOpenAIClient() + .GetProjectConversationsClient(); + + var conversation = (await conversationsClient.CreateProjectConversationAsync(options: null, cancellationToken).ConfigureAwait(false)).Value; + + return (ChatClientAgentSession)await ((ChatClientAgent)this.InnerAgent).CreateSessionAsync(conversation.Id, cancellationToken).ConfigureAwait(false); + } + + #endregion + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + if (serviceKey is null && serviceType == typeof(AIProjectClient)) + { + return this._aiProjectClient; + } + + return base.GetService(serviceType, serviceKey); + } + + #region Private helpers + + private static ChatClientAgent CreateInnerAgent( + AIProjectClient aiProjectClient, + string model, string instructions, + string? name, string? description, + IList? tools, + Func? clientFactory, + ILoggerFactory? loggerFactory, + IServiceProvider? services, + out AIProjectClient outClient) + { + Throw.IfNullOrWhitespace(model); + Throw.IfNullOrWhitespace(instructions); + + outClient = aiProjectClient; + + ChatClientAgentOptions options = new() + { + Name = name, + Description = description, + ChatOptions = new ChatOptions + { + ModelId = model, + Instructions = instructions, + Tools = tools, + }, + }; + + return AzureAIProjectChatClientExtensions.CreateResponsesChatClientAgent(aiProjectClient, options, clientFactory, loggerFactory, services); + } + + private static ChatClientAgent CreateInnerAgentFromEndpoint( + AIProjectClient aiProjectClient, + Uri agentEndpoint, + IList? tools, + Func? clientFactory, + IServiceProvider? services, + out AIProjectClient outClient) + { + outClient = aiProjectClient; + + AgentReference agentReference = agentEndpoint.Segments[^1].TrimEnd('/'); + + ChatClientAgentOptions agentOptions = new() + { + Name = agentReference.Name, + ChatOptions = new() { Tools = tools }, + }; + + IChatClient chatClient = new AzureAIProjectChatClient(aiProjectClient, agentReference, defaultModelId: null, agentOptions.ChatOptions); + + if (clientFactory is not null) + { + chatClient = clientFactory(chatClient); + } + + return new ChatClientAgent(chatClient, agentOptions, services: services); + } + + private static AIProjectClient CreateProjectClient(Uri endpoint, AuthenticationTokenProvider credential, AIProjectClientOptions? clientOptions = null) + { + Throw.IfNull(endpoint); + Throw.IfNull(credential); + + clientOptions ??= new AIProjectClientOptions(); + clientOptions.AddPolicy(RequestOptionsExtensions.UserAgentPolicy, System.ClientModel.Primitives.PipelinePosition.PerCall); + return new AIProjectClient(endpoint, credential, clientOptions); + } + + #endregion +} diff --git a/dotnet/src/Microsoft.Agents.AI.AzureAI/RequestOptionsExtensions.cs b/dotnet/src/Microsoft.Agents.AI.AzureAI/RequestOptionsExtensions.cs index 722d316330..2705611b57 100644 --- a/dotnet/src/Microsoft.Agents.AI.AzureAI/RequestOptionsExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.AzureAI/RequestOptionsExtensions.cs @@ -7,6 +7,9 @@ namespace Microsoft.Agents.AI; internal static class RequestOptionsExtensions { + /// Gets the singleton that adds a MEAI user-agent header. + internal static PipelinePolicy UserAgentPolicy => MeaiUserAgentPolicy.Instance; + /// Creates a configured for use with Foundry Agents. public static RequestOptions ToRequestOptions(this CancellationToken cancellationToken, bool streaming) { diff --git a/dotnet/src/Microsoft.Agents.AI.FoundryMemory/FoundryMemoryProvider.cs b/dotnet/src/Microsoft.Agents.AI.FoundryMemory/FoundryMemoryProvider.cs index 6f9f37518e..93b343b2de 100644 --- a/dotnet/src/Microsoft.Agents.AI.FoundryMemory/FoundryMemoryProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.FoundryMemory/FoundryMemoryProvider.cs @@ -19,7 +19,7 @@ using OpenAI.Responses; namespace Microsoft.Agents.AI.FoundryMemory; /// -/// Provides an Azure AI Foundry Memory backed that persists conversation messages as memories +/// Provides a Microsoft Foundry Memory backed that persists conversation messages as memories /// and retrieves related memories to augment the agent invocation context. /// /// @@ -49,7 +49,7 @@ public sealed class FoundryMemoryProvider : AIContextProvider /// Initializes a new instance of the class. /// /// The Azure AI Project client configured for your Foundry project. - /// The name of the memory store in Azure AI Foundry. + /// The name of the memory store in Microsoft Foundry. /// A delegate that initializes the provider state on the first invocation, providing the scope for memory storage and retrieval. /// Provider options. /// Optional logger factory. @@ -87,17 +87,7 @@ public sealed class FoundryMemoryProvider : AIContextProvider public override IReadOnlyList StateKeys => this._stateKeys ??= [this._sessionState.StateKey]; private static Func ValidateStateInitializer(Func stateInitializer) => - session => - { - State state = stateInitializer(session); - - if (state is null) - { - throw new InvalidOperationException("State initializer must return a non-null state."); - } - - return state; - }; + session => stateInitializer(session) ?? throw new InvalidOperationException("State initializer must return a non-null state."); /// protected override async ValueTask ProvideAIContextAsync(InvokingContext context, CancellationToken cancellationToken = default) @@ -332,7 +322,7 @@ public sealed class FoundryMemoryProvider : AIContextProvider /// Waits for all pending memory update operations to complete. /// /// - /// Memory extraction in Azure AI Foundry is asynchronous. This method polls the latest pending update + /// Memory extraction in Microsoft Foundry is asynchronous. This method polls the latest pending update /// and returns when it has completed, failed, or been superseded. Since updates are processed in order, /// completion of the latest update implies all prior updates have also been processed. /// diff --git a/dotnet/src/Microsoft.Agents.AI.FoundryMemory/FoundryMemoryProviderScope.cs b/dotnet/src/Microsoft.Agents.AI.FoundryMemory/FoundryMemoryProviderScope.cs index 717df1d12b..6646c482a3 100644 --- a/dotnet/src/Microsoft.Agents.AI.FoundryMemory/FoundryMemoryProviderScope.cs +++ b/dotnet/src/Microsoft.Agents.AI.FoundryMemory/FoundryMemoryProviderScope.cs @@ -9,7 +9,7 @@ namespace Microsoft.Agents.AI.FoundryMemory; /// Allows scoping of memories for the . /// /// -/// Azure AI Foundry memories are scoped by a single string identifier that you control. +/// Microsoft Foundry memories are scoped by a single string identifier that you control. /// Common patterns include using a user ID, team ID, or other unique identifier /// to partition memories across different contexts. /// diff --git a/dotnet/src/Microsoft.Agents.AI.OpenAI/CompatibilitySuppressions.xml b/dotnet/src/Microsoft.Agents.AI.OpenAI/CompatibilitySuppressions.xml new file mode 100644 index 0000000000..fe04c26eca --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.OpenAI/CompatibilitySuppressions.xml @@ -0,0 +1,109 @@ + + + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/net10.0/Microsoft.Agents.AI.OpenAI.dll + lib/net10.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/net10.0/Microsoft.Agents.AI.OpenAI.dll + lib/net10.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsIChatClientWithStoredOutputDisabled(OpenAI.Responses.ResponsesClient) + lib/net10.0/Microsoft.Agents.AI.OpenAI.dll + lib/net10.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/net472/Microsoft.Agents.AI.OpenAI.dll + lib/net472/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/net472/Microsoft.Agents.AI.OpenAI.dll + lib/net472/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsIChatClientWithStoredOutputDisabled(OpenAI.Responses.ResponsesClient) + lib/net472/Microsoft.Agents.AI.OpenAI.dll + lib/net472/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/net8.0/Microsoft.Agents.AI.OpenAI.dll + lib/net8.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/net8.0/Microsoft.Agents.AI.OpenAI.dll + lib/net8.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsIChatClientWithStoredOutputDisabled(OpenAI.Responses.ResponsesClient) + lib/net8.0/Microsoft.Agents.AI.OpenAI.dll + lib/net8.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/net9.0/Microsoft.Agents.AI.OpenAI.dll + lib/net9.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/net9.0/Microsoft.Agents.AI.OpenAI.dll + lib/net9.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsIChatClientWithStoredOutputDisabled(OpenAI.Responses.ResponsesClient) + lib/net9.0/Microsoft.Agents.AI.OpenAI.dll + lib/net9.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,Microsoft.Agents.AI.ChatClientAgentOptions,System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/netstandard2.0/Microsoft.Agents.AI.OpenAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsAIAgent(OpenAI.Responses.ResponsesClient,System.String,System.String,System.String,System.Collections.Generic.IList{Microsoft.Extensions.AI.AITool},System.Func{Microsoft.Extensions.AI.IChatClient,Microsoft.Extensions.AI.IChatClient},Microsoft.Extensions.Logging.ILoggerFactory,System.IServiceProvider) + lib/netstandard2.0/Microsoft.Agents.AI.OpenAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.OpenAI.dll + true + + + CP0002 + M:OpenAI.Responses.OpenAIResponseClientExtensions.AsIChatClientWithStoredOutputDisabled(OpenAI.Responses.ResponsesClient) + lib/netstandard2.0/Microsoft.Agents.AI.OpenAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.OpenAI.dll + true + + \ No newline at end of file diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/AzureAgentProvider.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/AzureAgentProvider.cs index bfc7bd36ff..6db870f8ec 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/AzureAgentProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/AzureAgentProvider.cs @@ -175,7 +175,7 @@ public sealed class AzureAgentProvider(Uri projectEndpoint, TokenCredential proj AIProjectClient client = this.GetAgentClient(); - agent = client.AsAIAgent(agentVersion, tools: null, clientFactory: null, services: null); + agent = client.AsAIAgent(agentVersion); FunctionInvokingChatClient? functionInvokingClient = agent.GetService(); if (functionInvokingClient is not null) diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/CompatibilitySuppressions.xml b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/CompatibilitySuppressions.xml new file mode 100644 index 0000000000..3454984fae --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.AzureAI/CompatibilitySuppressions.xml @@ -0,0 +1,39 @@ + + + + + CP0002 + M:Microsoft.Agents.AI.Workflows.Declarative.AzureAgentProvider.get_OpenAIClientOptions + lib/net10.0/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.Declarative.AzureAgentProvider.get_OpenAIClientOptions + lib/net472/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + lib/net472/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.Declarative.AzureAgentProvider.get_OpenAIClientOptions + lib/net8.0/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.Declarative.AzureAgentProvider.get_OpenAIClientOptions + lib/net9.0/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.Declarative.AzureAgentProvider.get_OpenAIClientOptions + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.Declarative.AzureAI.dll + true + + \ No newline at end of file diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.Mcp/Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.Mcp/Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj index f9bf706669..bca32e93fc 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.Mcp/Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative.Mcp/Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj @@ -13,6 +13,11 @@ + + + false + + Microsoft Agent Framework Declarative Workflows MCP diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/CompatibilitySuppressions.xml b/dotnet/src/Microsoft.Agents.AI.Workflows/CompatibilitySuppressions.xml new file mode 100644 index 0000000000..f227141706 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/CompatibilitySuppressions.xml @@ -0,0 +1,319 @@ + + + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config`1 + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.ConfigurationExtensions + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`1 + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`2 + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config`1 + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.ConfigurationExtensions + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`1 + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`2 + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config`1 + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.ConfigurationExtensions + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`1 + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`2 + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config`1 + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.ConfigurationExtensions + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`1 + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`2 + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Config`1 + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.ConfigurationExtensions + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`1 + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0001 + T:Microsoft.Agents.AI.Workflows.Configured`2 + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.AgentWorkflowBuilder.CreateHandoffBuilderWith(Microsoft.Agents.AI.AIAgent) + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.BindExecutor``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.ConfigureFactory``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + lib/net10.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.AgentWorkflowBuilder.CreateHandoffBuilderWith(Microsoft.Agents.AI.AIAgent) + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.BindExecutor``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.ConfigureFactory``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/net472/Microsoft.Agents.AI.Workflows.dll + lib/net472/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.AgentWorkflowBuilder.CreateHandoffBuilderWith(Microsoft.Agents.AI.AIAgent) + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.BindExecutor``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.ConfigureFactory``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + lib/net8.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.AgentWorkflowBuilder.CreateHandoffBuilderWith(Microsoft.Agents.AI.AIAgent) + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.BindExecutor``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.ConfigureFactory``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + lib/net9.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.AgentWorkflowBuilder.CreateHandoffBuilderWith(Microsoft.Agents.AI.AIAgent) + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.BindExecutor``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + + CP0002 + M:Microsoft.Agents.AI.Workflows.ExecutorBindingExtensions.ConfigureFactory``2(System.Func{Microsoft.Agents.AI.Workflows.Config{``1},System.String,System.Threading.Tasks.ValueTask{``0}},System.String,``1) + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + lib/netstandard2.0/Microsoft.Agents.AI.Workflows.dll + true + + \ No newline at end of file diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgent.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgent.cs index 05caff4d83..713361d9be 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgent.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgent.cs @@ -139,8 +139,8 @@ public sealed partial class ChatClientAgent : AIAgent this._logger = (loggerFactory ?? chatClient.GetService() ?? NullLoggerFactory.Instance).CreateLogger(); - // Warn if using a custom chat client stack with end-of-run persistence but no ChatHistoryPersistingChatClient. - this.WarnOnMissingPersistingClient(); + // Warn if using a custom chat client stack with simulated service stored persistence but no ServiceStoredSimulatingChatClient. + this.WarnOnMissingServiceStoredSimulatingClient(); } /// @@ -454,7 +454,7 @@ public sealed partial class ChatClientAgent : AIAgent /// Notifies the and all of successfully completed messages. /// /// - /// This method is also called by to persist messages per-service-call. + /// This method is also called by to persist messages per-service-call. /// internal async Task NotifyProvidersOfNewMessagesAsync( ChatClientAgentSession session, @@ -463,7 +463,7 @@ public sealed partial class ChatClientAgent : AIAgent ChatOptions? chatOptions, CancellationToken cancellationToken) { - ChatHistoryProvider? chatHistoryProvider = this.ResolveChatHistoryProvider(chatOptions, session); + ChatHistoryProvider? chatHistoryProvider = this.ResolveChatHistoryProvider(chatOptions); if (chatHistoryProvider is not null) { @@ -486,7 +486,7 @@ public sealed partial class ChatClientAgent : AIAgent /// Notifies the and all of a failure during a service call. /// /// - /// This method is also called by to report failures per-service-call. + /// This method is also called by to report failures per-service-call. /// internal async Task NotifyProvidersOfFailureAsync( ChatClientAgentSession session, @@ -495,7 +495,7 @@ public sealed partial class ChatClientAgent : AIAgent ChatOptions? chatOptions, CancellationToken cancellationToken) { - ChatHistoryProvider? chatHistoryProvider = this.ResolveChatHistoryProvider(chatOptions, session); + ChatHistoryProvider? chatHistoryProvider = this.ResolveChatHistoryProvider(chatOptions); if (chatHistoryProvider is not null) { @@ -701,7 +701,7 @@ public sealed partial class ChatClientAgent : AIAgent throw new InvalidOperationException("A session must be provided when continuing a background response with a continuation token."); } - if ((continuationToken is not null || chatOptions?.AllowBackgroundResponses is true) && this.PersistsChatHistoryPerServiceCall && this._logger.IsEnabled(LogLevel.Warning)) + if ((continuationToken is not null || chatOptions?.AllowBackgroundResponses is true) && this.SimulatesServiceStoredChatHistory && this._logger.IsEnabled(LogLevel.Warning)) { var warningAgentName = this.GetLoggingAgentName(); this._logger.LogAgentChatClientBackgroundResponseFallback(this.Id, warningAgentName); @@ -719,57 +719,6 @@ public sealed partial class ChatClientAgent : AIAgent throw new InvalidOperationException("Input messages are not allowed when continuing a background response using a continuation token."); } - IEnumerable inputMessagesForChatClient = inputMessages; - - // Populate the session messages only if we are not continuing an existing response as it's not allowed - if (chatOptions?.ContinuationToken is null) - { - ChatHistoryProvider? chatHistoryProvider = this.ResolveChatHistoryProvider(chatOptions, typedSession); - - // Add any existing messages from the session to the messages to be sent to the chat client. - // The ChatHistoryProvider returns the merged result (history + input messages). - if (chatHistoryProvider is not null) - { - var invokingContext = new ChatHistoryProvider.InvokingContext(this, typedSession, inputMessagesForChatClient); - inputMessagesForChatClient = await chatHistoryProvider.InvokingAsync(invokingContext, cancellationToken).ConfigureAwait(false); - } - - // If we have an AIContextProvider, we should get context from it, and update our - // messages and options with the additional context. - // The AIContextProvider returns the accumulated AIContext (original + new contributions). - if (this.AIContextProviders is { Count: > 0 } aiContextProviders) - { - var aiContext = new AIContext - { - Instructions = chatOptions?.Instructions, - Messages = inputMessagesForChatClient, - Tools = chatOptions?.Tools - }; - - foreach (var aiContextProvider in aiContextProviders) - { - var invokingContext = new AIContextProvider.InvokingContext(this, typedSession, aiContext); - aiContext = await aiContextProvider.InvokingAsync(invokingContext, cancellationToken).ConfigureAwait(false); - } - - // Materialize the accumulated messages and tools once at the end of the provider pipeline. - inputMessagesForChatClient = aiContext.Messages ?? []; - - var tools = aiContext.Tools as IList ?? aiContext.Tools?.ToList(); - if (chatOptions?.Tools is { Count: > 0 } || tools is { Count: > 0 }) - { - chatOptions ??= new(); - chatOptions.Tools = tools; - } - - if (chatOptions?.Instructions is not null || aiContext.Instructions is not null) - { - chatOptions ??= new(); - chatOptions.Instructions = aiContext.Instructions; - } - } - } - // If a user provided two different session ids, via the session object and options, we should throw // since we don't know which one to use. if (!string.IsNullOrWhiteSpace(typedSession.ConversationId) && !string.IsNullOrWhiteSpace(chatOptions?.ConversationId) && typedSession.ConversationId != chatOptions!.ConversationId) @@ -788,12 +737,53 @@ public sealed partial class ChatClientAgent : AIAgent chatOptions.ConversationId = typedSession.ConversationId; } - // When per-service-call persistence is active, set a sentinel conversation ID so that - // FunctionInvokingChatClient treats locally-persisted history the same as service-managed - // history. This prevents it from adding duplicate FunctionCallContent messages into the - // request when processing approval responses — the loaded history already contains them. - // ChatHistoryPersistingChatClient strips the sentinel before forwarding to the inner client. - chatOptions = this.SetLocalHistoryConversationIdIfNeeded(chatOptions); + IEnumerable inputMessagesForChatClient = inputMessages; + + // Populate the session messages only if we are not continuing an existing response as it's not allowed. + // When SimulateServiceStoredChatHistory is active, the ServiceStoredSimulatingChatClient + // owns the chat history lifecycle — it loads history before each service call. The agent + // must not load history itself, as that would result in duplicate messages. + if (chatOptions?.ContinuationToken is null && !this.SimulatesServiceStoredChatHistory) + { + // Add any existing messages from the session to the messages to be sent to the chat client. + // The ChatHistoryProvider returns the merged result (history + input messages). + inputMessagesForChatClient = await this.LoadChatHistoryAsync(typedSession, inputMessagesForChatClient, chatOptions, cancellationToken).ConfigureAwait(false); + } + + // AIContextProviders should always be invoked (unless continuing an existing response) + // to contribute additional messages, tools, and instructions — even when the decorator + // handles history loading. + if (chatOptions?.ContinuationToken is null && this.AIContextProviders is { Count: > 0 } aiContextProviders) + { + var aiContext = new AIContext + { + Instructions = chatOptions?.Instructions, + Messages = inputMessagesForChatClient, + Tools = chatOptions?.Tools + }; + + foreach (var aiContextProvider in aiContextProviders) + { + var invokingContext = new AIContextProvider.InvokingContext(this, typedSession, aiContext); + aiContext = await aiContextProvider.InvokingAsync(invokingContext, cancellationToken).ConfigureAwait(false); + } + + // Materialize the accumulated messages and tools once at the end of the provider pipeline. + inputMessagesForChatClient = aiContext.Messages ?? []; + + var tools = aiContext.Tools as IList ?? aiContext.Tools?.ToList(); + if (chatOptions?.Tools is { Count: > 0 } || tools is { Count: > 0 }) + { + chatOptions ??= new(); + chatOptions.Tools = tools; + } + + if (chatOptions?.Instructions is not null || aiContext.Instructions is not null) + { + chatOptions ??= new(); + chatOptions.Instructions = aiContext.Instructions; + } + } // Materialize the accumulated messages once at the end of the provider pipeline, reusing the existing list if possible. List messagesList = inputMessagesForChatClient as List ?? inputMessagesForChatClient.ToList(); @@ -839,8 +829,6 @@ public sealed partial class ChatClientAgent : AIAgent } } - // If we got a conversation id back from the chat client, it means that the service supports server side session storage - // so we should update the session with the new id. session.ConversationId = responseConversationId; } } @@ -849,14 +837,14 @@ public sealed partial class ChatClientAgent : AIAgent /// Updates the session conversation ID at the end of an agent run. /// /// - /// When a in persist mode handles per-service-call - /// conversation ID updates, this end-of-run update is skipped. When the decorator is in mark-only - /// mode or absent, the update is performed here. When is + /// When a handles per-service-call + /// conversation ID updates, this end-of-run update is skipped. When the decorator is + /// absent, the update is performed here. When is /// (continuation token scenarios), the update is always performed. /// private void UpdateSessionConversationIdAtEndOfRun(ChatClientAgentSession session, string? responseConversationId, CancellationToken cancellationToken, bool forceUpdate = false) { - if (!forceUpdate && this.PersistsChatHistoryPerServiceCall) + if (!forceUpdate && this.SimulatesServiceStoredChatHistory) { return; } @@ -868,10 +856,9 @@ public sealed partial class ChatClientAgent : AIAgent /// Notifies providers of successfully completed messages at the end of an agent run. /// /// - /// When a in persist mode handles per-service-call - /// notification, this end-of-run notification is skipped. When the decorator is in mark-only mode, - /// only the marked messages are persisted. When no decorator is present (custom stack with - /// ), all messages are persisted. + /// When a handles per-service-call + /// notification, this end-of-run notification is skipped. When no decorator is present, + /// all messages are persisted. /// When is (continuation token or /// background response scenarios), notification is always performed with all messages because /// per-service-call persistence is unreliable in these scenarios. @@ -884,19 +871,11 @@ public sealed partial class ChatClientAgent : AIAgent CancellationToken cancellationToken, bool forceNotify = false) { - if (!forceNotify && this.PersistsChatHistoryPerServiceCall) + if (!forceNotify && this.SimulatesServiceStoredChatHistory) { return Task.CompletedTask; } - if (!forceNotify && this.HasMarkOnlyChatHistoryPersistingClient) - { - // In mark-only mode, persist only messages that were marked by the decorator. - var markedRequestMessages = GetMarkedMessages(requestMessages); - var markedResponseMessages = GetMarkedMessages(responseMessages); - return this.NotifyProvidersOfNewMessagesAsync(session, markedRequestMessages, markedResponseMessages, chatOptions, cancellationToken); - } - return this.NotifyProvidersOfNewMessagesAsync(session, requestMessages, responseMessages, chatOptions, cancellationToken); } @@ -904,7 +883,7 @@ public sealed partial class ChatClientAgent : AIAgent /// Notifies providers of a failure at the end of an agent run. /// /// - /// When a in persist mode handles per-service-call + /// When a handles per-service-call /// notification (including failure), this end-of-run notification is skipped to avoid /// duplicate notification. In all other cases, failure is reported at the end of the run. /// @@ -915,7 +894,7 @@ public sealed partial class ChatClientAgent : AIAgent ChatOptions? chatOptions, CancellationToken cancellationToken) { - if (this.PersistsChatHistoryPerServiceCall) + if (this.SimulatesServiceStoredChatHistory) { return Task.CompletedTask; } @@ -924,60 +903,19 @@ public sealed partial class ChatClientAgent : AIAgent } /// - /// Gets a value indicating whether the agent has a - /// decorator in persist mode (not mark-only), which handles per-service-call persistence. + /// Gets a value indicating whether the agent is configured to simulate service-stored chat history. + /// When , end-of-run persistence and history loading are skipped because a + /// per-service-call decorator (such as or a + /// user-supplied equivalent) is expected to handle the history lifecycle. /// - private bool PersistsChatHistoryPerServiceCall + private bool SimulatesServiceStoredChatHistory { get { - var persistingClient = this.ChatClient.GetService(); - return persistingClient?.MarkOnly == false; + return this._agentOptions?.SimulateServiceStoredChatHistory is true; } } - /// - /// Sets the sentinel on - /// when per-service-call persistence is active and no real - /// conversation ID is present. - /// - /// - /// The (possibly new) with the sentinel set, or the original - /// if no sentinel is needed. - /// - private ChatOptions? SetLocalHistoryConversationIdIfNeeded(ChatOptions? chatOptions) - { - if (this.PersistsChatHistoryPerServiceCall && string.IsNullOrWhiteSpace(chatOptions?.ConversationId)) - { - chatOptions ??= new ChatOptions(); - chatOptions.ConversationId = ChatHistoryPersistingChatClient.LocalHistoryConversationId; - } - - return chatOptions; - } - - /// - /// Gets a value indicating whether the agent has a - /// decorator in mark-only mode, which marks messages for later persistence at the end of the run. - /// - private bool HasMarkOnlyChatHistoryPersistingClient - { - get - { - var persistingClient = this.ChatClient.GetService(); - return persistingClient?.MarkOnly == true; - } - } - - /// - /// Returns only the messages that have been marked as persisted by a in mark-only mode. - /// - private static List GetMarkedMessages(IEnumerable messages) - { - return messages.Where(m => - m.AdditionalProperties?.TryGetValue(ChatHistoryPersistingChatClient.PersistedMarkerKey, out var value) == true && value is true).ToList(); - } - /// /// Ensures that contains the resolved session. /// @@ -985,7 +923,7 @@ public sealed partial class ChatClientAgent : AIAgent /// The base class sets with the raw session parameter /// (which may be null) and restores it after each yield in streaming scenarios. After /// resolves or creates a session, we update the - /// context so the decorator always has a valid session. + /// context so the decorator always has a valid session. /// The original agent from the context is preserved to maintain the top-of-stack agent in /// decorated agent scenarios. /// @@ -1001,36 +939,36 @@ public sealed partial class ChatClientAgent : AIAgent /// /// Checks for potential misconfiguration when using a custom chat client stack and logs warnings. /// - private void WarnOnMissingPersistingClient() + private void WarnOnMissingServiceStoredSimulatingClient() { if (this._agentOptions?.UseProvidedChatClientAsIs is not true) { return; } - if (this._agentOptions?.PersistChatHistoryAtEndOfRun is not true) + if (this._agentOptions?.SimulateServiceStoredChatHistory is not true) { return; } - var persistingClient = this.ChatClient.GetService(); + var persistingClient = this.ChatClient.GetService(); if (persistingClient is null && this._logger.IsEnabled(LogLevel.Warning)) { var loggingAgentName = this.GetLoggingAgentName(); this._logger.LogAgentChatClientMissingPersistingClient( this.Id, - loggingAgentName); + loggingAgentName); // CodeQL [CWE-359] False positive: Agent name is not personal information, but rather just the name of a code component (agent in this case). } } - private ChatHistoryProvider? ResolveChatHistoryProvider(ChatOptions? chatOptions, ChatClientAgentSession session) + private ChatHistoryProvider? ResolveChatHistoryProvider(ChatOptions? chatOptions) { - ChatHistoryProvider? provider = session.ConversationId is null ? this.ChatHistoryProvider : null; + ChatHistoryProvider? provider = chatOptions?.ConversationId is null ? this.ChatHistoryProvider : null; // If someone provided an override ChatHistoryProvider via AdditionalProperties, we should use that instead. if (chatOptions?.AdditionalProperties?.TryGetValue(out ChatHistoryProvider? overrideProvider) is true) { - if (session.ConversationId is not null && overrideProvider is not null) + if (this._agentOptions?.ThrowOnChatHistoryProviderConflict is true && string.IsNullOrWhiteSpace(chatOptions?.ConversationId) is false) { throw new InvalidOperationException( $"Only {nameof(ChatClientAgentSession.ConversationId)} or {nameof(this.ChatHistoryProvider)} may be used, but not both. The current {nameof(ChatClientAgentSession)} has a {nameof(ChatClientAgentSession.ConversationId)} indicating server-side chat history management, but an override {nameof(this.ChatHistoryProvider)} was provided via {nameof(AgentRunOptions.AdditionalProperties)}."); @@ -1055,6 +993,29 @@ public sealed partial class ChatClientAgent : AIAgent return provider; } + /// + /// Loads chat history from the resolved and prepends it to the given messages. + /// + /// + /// This method is used by both the agent (during ) and by + /// to load history before each service call. + /// + internal async Task> LoadChatHistoryAsync( + ChatClientAgentSession session, + IEnumerable messages, + ChatOptions? chatOptions, + CancellationToken cancellationToken) + { + var chatHistoryProvider = this.ResolveChatHistoryProvider(chatOptions); + if (chatHistoryProvider is null) + { + return messages; + } + + var invokingContext = new ChatHistoryProvider.InvokingContext(this, session, messages); + return await chatHistoryProvider.InvokingAsync(invokingContext, cancellationToken).ConfigureAwait(false); + } + private static ChatClientAgentContinuationToken? WrapContinuationToken(ResponseContinuationToken? continuationToken, IEnumerable? inputMessages = null, List? responseUpdates = null) { if (continuationToken is null) diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgentLogMessages.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgentLogMessages.cs index 2a324522a4..5899ef5cb2 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgentLogMessages.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgentLogMessages.cs @@ -72,12 +72,12 @@ internal static partial class ChatClientAgentLogMessages /// /// Logs a warning when is - /// and is , - /// but no is found in the custom chat client stack. + /// and is , + /// but no is found in the custom chat client stack. /// [LoggerMessage( Level = LogLevel.Warning, - Message = "Agent {AgentId}/{AgentName}: PersistChatHistoryAtEndOfRun is enabled with a custom chat client stack (UseProvidedChatClientAsIs), but no ChatHistoryPersistingChatClient was found in the pipeline. All messages will be persisted at the end of the run without marking. This setup is not supported with some other features, e.g. handoffs. Consider adding a ChatHistoryPersistingChatClient to the pipeline using the UseChatHistoryPersisting extension method.")] + Message = "Agent {AgentId}/{AgentName}: SimulateServiceStoredChatHistory is enabled with a custom chat client stack (UseProvidedChatClientAsIs), but no ServiceStoredSimulatingChatClient was found in the pipeline. Chat history will not be persisted by ChatClientAgent. Consider adding a ServiceStoredSimulatingChatClient to the pipeline using the UseServiceStoredChatHistorySimulation extension method if you have not added your own persistence mechanism.")] public static partial void LogAgentChatClientMissingPersistingClient( this ILogger logger, string agentId, @@ -92,7 +92,7 @@ internal static partial class ChatClientAgentLogMessages /// [LoggerMessage( Level = LogLevel.Warning, - Message = "Agent {AgentId}/{AgentName}: Per-service-call persistence is falling back to end-of-run persistence because the run involves background responses. Messages will be marked during the run and persisted at the end.")] + Message = "Agent {AgentId}/{AgentName}: SimulateServiceStoredChatHistory is enabled but we have to fall back to end-of-run persistence because the run involves background responses.")] public static partial void LogAgentChatClientBackgroundResponseFallback( this ILogger logger, string agentId, diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgentOptions.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgentOptions.cs index 8df9112446..e3d4326ae6 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgentOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientAgentOptions.cs @@ -92,54 +92,46 @@ public sealed class ChatClientAgentOptions public bool ThrowOnChatHistoryProviderConflict { get; set; } = true; /// - /// Gets or sets a value indicating whether to persist chat history only at the end of the full agent run - /// rather than after each individual service call. + /// Gets or sets a value indicating whether the should simulate + /// service-stored chat history behavior using its configured . /// /// /// - /// By default, persists request and response messages either via - /// a , or the underlying AI service's chat history storage. - /// Persistence is done immediately after each call to the AI service within the function invocation loop. - /// When storing in the underlying AI service, the session's - /// is also updated after each service call, keeping it in sync with the service-side conversation state. + /// When set to , a decorator is + /// injected between the and the leaf + /// in the chat client pipeline. This decorator takes full ownership of the chat history lifecycle: + /// it loads history from the before each service call and persists + /// new messages after each service call. It also returns a sentinel + /// on the response, causing the to treat the conversation + /// as service-managed — clearing accumulated history and not injecting duplicate + /// during approval-response processing. /// /// - /// Setting this property to causes messages to be marked during the function - /// invocation loop but persisted only at the end of the full agent run, providing atomic run semantics. - /// Updating the is likewise deferred and - /// updated only at the end of the run, consistent with atomic run semantics. - /// A decorator is inserted into the chat client pipeline - /// in mark-only mode, and the persists only the marked messages at the - /// end of the run. + /// This mode aligns the behavior of framework-managed chat history with service-stored chat history, + /// ensuring consistency in how messages are stored and loaded, including during function calling loops + /// and tool-call termination scenarios. /// /// - /// When this option is (the default), the - /// decorator persists messages and updates the - /// immediately after each service call. This may leave chat history in a state where - /// is required to start a new run if the last successful service - /// call returned . + /// When set to (the default), the handles + /// chat history persistence at the end of the full agent run via the + /// pipeline. /// /// - /// This option has no effect when is . - /// When using a custom chat client stack, you can add a - /// manually via the + /// When setting the setting to and + /// to , ensure that your custom chat client stack includes a + /// to enable per-service-call persistence. + /// If no is provided, and you are not storing chat history via other means, + /// no chat history may be stored. + /// When using a custom chat client stack, you can add a + /// manually via the /// extension method. /// - /// - /// Note that when using single threaded service stored chat history, like OpenAI Conversations, - /// there is only one id, so even if the conversation id is not updated after each service call, - /// the chat history will still contain intermediate messages. Setting this property to - /// in this case will therefore have no real effect. Setting this property to when using - /// OpenAI Responses with response ids on the other hand, allows atomic run semantics, since - /// each service request produces a new response id, and if the run fails mid-loop, the session will - /// still contain the pre-run respnose id, allowing the next run to start with a clean slate. - /// /// /// /// Default is . /// [Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] - public bool PersistChatHistoryAtEndOfRun { get; set; } + public bool SimulateServiceStoredChatHistory { get; set; } /// /// Creates a new instance of with the same values as this instance. @@ -157,6 +149,6 @@ public sealed class ChatClientAgentOptions ClearOnChatHistoryProviderConflict = this.ClearOnChatHistoryProviderConflict, WarnOnChatHistoryProviderConflict = this.WarnOnChatHistoryProviderConflict, ThrowOnChatHistoryProviderConflict = this.ThrowOnChatHistoryProviderConflict, - PersistChatHistoryAtEndOfRun = this.PersistChatHistoryAtEndOfRun, + SimulateServiceStoredChatHistory = this.SimulateServiceStoredChatHistory, }; } diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientBuilderExtensions.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientBuilderExtensions.cs index a1e8b5f8a5..68e1307eeb 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientBuilderExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientBuilderExtensions.cs @@ -86,25 +86,21 @@ public static class ChatClientBuilderExtensions services: services); /// - /// Adds a to the chat client pipeline. + /// Adds a to the chat client pipeline. /// /// /// /// This decorator should be positioned between the and the leaf - /// in the pipeline. It intercepts service calls to either persist messages - /// immediately or mark them for later persistence, depending on the parameter. - /// - /// - /// If is set to , the - /// should be configured with set to - /// as without this combination, messages will never be persisted when using a for - /// chat history persistence. + /// in the pipeline. It simulates service-stored chat history behavior by + /// loading history before each service call, persisting after each call, and returning a sentinel + /// on the response. /// /// /// This extension method is intended for use with custom chat client stacks when /// is . /// When is (the default), - /// the automatically injects this decorator. + /// the automatically injects this decorator when + /// is . /// /// /// This decorator only works within the context of a running and will throw an @@ -112,18 +108,10 @@ public static class ChatClientBuilderExtensions /// /// /// The to add the decorator to. - /// - /// When , messages are marked with metadata but not persisted immediately, - /// and the session's is not updated. - /// The will persist only the marked messages and update the - /// conversation ID at the end of the run. - /// When (the default), messages are persisted and the conversation ID - /// is updated immediately after each service call. - /// /// The for chaining. [Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] - public static ChatClientBuilder UseChatHistoryPersisting(this ChatClientBuilder builder, bool markOnly = false) + public static ChatClientBuilder UseServiceStoredChatHistorySimulation(this ChatClientBuilder builder) { - return builder.Use(innerClient => new ChatHistoryPersistingChatClient(innerClient, markOnly)); + return builder.Use(innerClient => new ServiceStoredSimulatingChatClient(innerClient)); } } diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientExtensions.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientExtensions.cs index fffac628a6..5251314766 100644 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatClientExtensions.cs @@ -63,14 +63,17 @@ public static class ChatClientExtensions }); } - // ChatHistoryPersistingChatClient is registered after FunctionInvokingChatClient so that it sits - // between FIC and the leaf client. ChatClientBuilder.Build applies factories in reverse order, - // making the first Use() call outermost. By adding our decorator second, the resulting pipeline is: - // FunctionInvokingChatClient → ChatHistoryPersistingChatClient → leaf IChatClient - // This allows the decorator to persist messages after each individual service call within - // FIC's function invocation loop, or to mark them for later persistence at the end of the run. - bool markOnly = options?.PersistChatHistoryAtEndOfRun is true; - chatBuilder.Use(innerClient => new ChatHistoryPersistingChatClient(innerClient, markOnly)); + // ServiceStoredSimulatingChatClient is only injected when SimulateServiceStoredChatHistory is enabled. + // It is registered after FunctionInvokingChatClient so that it sits between FIC and the leaf client. + // ChatClientBuilder.Build applies factories in reverse order, making the first Use() call outermost. + // By adding our decorator second, the resulting pipeline is: + // FunctionInvokingChatClient → ServiceStoredSimulatingChatClient → leaf IChatClient + // This allows the decorator to simulate service-stored chat history by loading history before + // each service call, persisting after each call, and returning a sentinel ConversationId. + if (options?.SimulateServiceStoredChatHistory is true) + { + chatBuilder.Use(innerClient => new ServiceStoredSimulatingChatClient(innerClient)); + } var agentChatClient = chatBuilder.Build(services); diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatHistoryPersistingChatClient.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatHistoryPersistingChatClient.cs deleted file mode 100644 index e733b778eb..0000000000 --- a/dotnet/src/Microsoft.Agents.AI/ChatClient/ChatHistoryPersistingChatClient.cs +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI; - -/// -/// A delegating chat client that notifies and -/// instances of request and response messages after each individual call to the inner chat client, -/// or marks messages for later persistence depending on the configured mode. -/// -/// -/// -/// This decorator is intended to operate between the and the leaf -/// in a pipeline. -/// -/// -/// In persist mode (the default), it ensures that providers are notified and the session's -/// is updated after each service call, so that -/// intermediate messages (e.g., tool calls and results) are saved even if the process is interrupted -/// mid-loop. -/// -/// -/// In mark-only mode ( is ), it marks messages with metadata -/// but does not notify providers or update the . -/// Both are deferred to the at the end of the run, providing atomic -/// run semantics. -/// -/// -/// This chat client must be used within the context of a running . It retrieves the -/// current agent and session from , which is set automatically when an agent's -/// or -/// -/// method is called. The ensures the run context always contains a resolved session, -/// even when the caller passes null. An is thrown if no run context is -/// available or if the agent is not a . -/// -/// -internal sealed class ChatHistoryPersistingChatClient : DelegatingChatClient -{ - /// - /// The key used in and - /// to mark messages and their content as already persisted to chat history. - /// - internal const string PersistedMarkerKey = "_chatHistoryPersisted"; - - /// - /// A sentinel value set on by - /// when per-service-call persistence is active and no real conversation ID exists. - /// - /// - /// - /// This signals to that the chat history is being managed - /// externally (by this decorator), which prevents it from adding duplicate - /// messages into the request during approval-response processing. Without this sentinel, - /// would reconstruct function-call messages from approval - /// responses and append them to the original messages — but the loaded history already contains - /// those same function calls, causing duplicate tool-call entries that the model rejects. - /// - /// - /// This decorator strips the sentinel before forwarding requests to the inner client, so the - /// underlying model never sees it. - /// - /// - internal const string LocalHistoryConversationId = "_agent_local_history"; - - /// - /// Initializes a new instance of the class. - /// - /// The underlying chat client that will handle the core operations. - /// - /// When , messages are marked with metadata but not persisted immediately, - /// and the session's is not updated. - /// The will persist only the marked messages and update the - /// conversation ID at the end of the run. - /// When (the default), messages are persisted and the conversation ID - /// is updated immediately after each service call. - /// - public ChatHistoryPersistingChatClient(IChatClient innerClient, bool markOnly = false) - : base(innerClient) - { - this.MarkOnly = markOnly; - } - - /// - /// Gets a value indicating whether this decorator is in mark-only mode. - /// - /// - /// When , messages are marked with metadata but not persisted immediately, - /// and the session's is not updated. - /// Both are deferred to the at the end of the run. - /// When , messages are persisted and the conversation ID is updated - /// after each service call. - /// - public bool MarkOnly { get; } - - /// - public override async Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - { - var (agent, session) = GetRequiredAgentAndSession(); - options = StripLocalHistoryConversationId(options); - - ChatResponse response; - try - { - response = await base.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - var newRequestMessagesOnFailure = GetNewRequestMessages(messages); - await agent.NotifyProvidersOfFailureAsync(session, ex, newRequestMessagesOnFailure, options, cancellationToken).ConfigureAwait(false); - throw; - } - - var newRequestMessages = GetNewRequestMessages(messages); - - if (this.ShouldDeferPersistence(options)) - { - // In mark-only mode or when resuming from a continuation token, just mark messages - // for later persistence by ChatClientAgent. Conversation ID and provider notification - // are deferred to end-of-run. For continuation tokens, the end-of-run handler needs - // to send the combined data from both the previous and current runs. - MarkAsPersisted(newRequestMessages); - MarkAsPersisted(response.Messages); - } - else - { - // In persist mode, persist immediately and update conversation ID. - agent.UpdateSessionConversationId(session, response.ConversationId, cancellationToken); - await agent.NotifyProvidersOfNewMessagesAsync(session, newRequestMessages, response.Messages, options, cancellationToken).ConfigureAwait(false); - MarkAsPersisted(newRequestMessages); - MarkAsPersisted(response.Messages); - } - - return response; - } - - /// - public override async IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - var (agent, session) = GetRequiredAgentAndSession(); - options = StripLocalHistoryConversationId(options); - - List responseUpdates = []; - - IAsyncEnumerator enumerator; - try - { - enumerator = base.GetStreamingResponseAsync(messages, options, cancellationToken).GetAsyncEnumerator(cancellationToken); - } - catch (Exception ex) - { - var newRequestMessagesOnFailure = GetNewRequestMessages(messages); - await agent.NotifyProvidersOfFailureAsync(session, ex, newRequestMessagesOnFailure, options, cancellationToken).ConfigureAwait(false); - throw; - } - - bool hasUpdates; - try - { - hasUpdates = await enumerator.MoveNextAsync().ConfigureAwait(false); - } - catch (Exception ex) - { - var newRequestMessagesOnFailure = GetNewRequestMessages(messages); - await agent.NotifyProvidersOfFailureAsync(session, ex, newRequestMessagesOnFailure, options, cancellationToken).ConfigureAwait(false); - throw; - } - - while (hasUpdates) - { - var update = enumerator.Current; - responseUpdates.Add(update); - yield return update; - - try - { - hasUpdates = await enumerator.MoveNextAsync().ConfigureAwait(false); - } - catch (Exception ex) - { - var newRequestMessagesOnFailure = GetNewRequestMessages(messages); - await agent.NotifyProvidersOfFailureAsync(session, ex, newRequestMessagesOnFailure, options, cancellationToken).ConfigureAwait(false); - throw; - } - } - - var chatResponse = responseUpdates.ToChatResponse(); - var newRequestMessages = GetNewRequestMessages(messages); - - if (this.ShouldDeferPersistence(options)) - { - // In mark-only mode or when resuming from a continuation token, just mark messages - // for later persistence by ChatClientAgent. Conversation ID and provider notification - // are deferred to end-of-run. For continuation tokens, the end-of-run handler needs - // to send the combined data from both the previous and current runs. - MarkAsPersisted(newRequestMessages); - MarkAsPersisted(chatResponse.Messages); - } - else - { - // In persist mode, persist immediately and update conversation ID. - agent.UpdateSessionConversationId(session, chatResponse.ConversationId, cancellationToken); - await agent.NotifyProvidersOfNewMessagesAsync(session, newRequestMessages, chatResponse.Messages, options, cancellationToken).ConfigureAwait(false); - MarkAsPersisted(newRequestMessages); - MarkAsPersisted(chatResponse.Messages); - } - } - - /// - /// Gets the current and from the run context. - /// - private static (ChatClientAgent Agent, ChatClientAgentSession Session) GetRequiredAgentAndSession() - { - var runContext = AIAgent.CurrentRunContext - ?? throw new InvalidOperationException( - $"{nameof(ChatHistoryPersistingChatClient)} can only be used within the context of a running AIAgent. " + - "Ensure that the chat client is being invoked as part of an AIAgent.RunAsync or AIAgent.RunStreamingAsync call."); - - var chatClientAgent = runContext.Agent.GetService() - ?? throw new InvalidOperationException( - $"{nameof(ChatHistoryPersistingChatClient)} can only be used with a {nameof(ChatClientAgent)}. " + - $"The current agent is of type '{runContext.Agent.GetType().Name}'."); - - if (runContext.Session is not ChatClientAgentSession chatClientAgentSession) - { - throw new InvalidOperationException( - $"{nameof(ChatHistoryPersistingChatClient)} requires a {nameof(ChatClientAgentSession)}. " + - $"The current session is of type '{runContext.Session?.GetType().Name ?? "null"}'."); - } - - return (chatClientAgent, chatClientAgentSession); - } - - /// - /// Determines whether persistence should be deferred to end-of-run instead of happening immediately. - /// - /// - /// when in mode, when the call is resuming from - /// a continuation token (since the end-of-run handler needs to combine data from the previous - /// and current runs), or when background responses are allowed (since the caller may stop - /// consuming the stream mid-run, preventing the post-stream persistence code from executing). - /// - private bool ShouldDeferPersistence(ChatOptions? options) - { - return this.MarkOnly || options?.ContinuationToken is not null || options?.AllowBackgroundResponses is true; - } - - /// - /// Returns only the request messages that have not yet been persisted to chat history. - /// - /// - /// A message is considered already persisted if any of the following is true: - /// - /// It has the in its . - /// It has an of - /// (indicating it was loaded from chat history and does not need to be re-persisted). - /// It has and all of its items have the - /// in their . This handles the - /// streaming case where reconstructs objects - /// independently via ToChatResponse(), producing different object references that share the same - /// underlying instances. - /// - /// - /// A list of request messages that have not yet been persisted. - /// The full set of request messages to filter. - private static List GetNewRequestMessages(IEnumerable messages) - { - return messages.Where(m => !IsAlreadyPersisted(m)).ToList(); - } - - /// - /// Determines whether a message has already been persisted to chat history by this decorator. - /// - private static bool IsAlreadyPersisted(ChatMessage message) - { - if (message.AdditionalProperties?.TryGetValue(PersistedMarkerKey, out var value) == true && value is true) - { - return true; - } - - if (message.GetAgentRequestMessageSourceType() == AgentRequestMessageSourceType.ChatHistory) - { - return true; - } - - // In streaming mode, FunctionInvokingChatClient reconstructs ChatMessage objects via ToChatResponse() - // independently, producing different ChatMessage instances. However, the underlying AIContent objects - // (e.g., FunctionCallContent, FunctionResultContent) are shared references. Checking for markers on - // AIContent handles dedup in this case. - if (message.Contents.Count > 0 && message.Contents.All(c => c.AdditionalProperties?.TryGetValue(PersistedMarkerKey, out var value) == true && value is true)) - { - return true; - } - - return false; - } - - /// - /// Marks the given messages as persisted by setting a marker on both the - /// and each of its items. - /// - /// - /// Both levels are marked because may reconstruct - /// objects in streaming mode (losing the message-level marker), - /// but the references are shared and retain their markers. - /// - /// The messages to mark as persisted. - private static void MarkAsPersisted(IEnumerable messages) - { - foreach (var message in messages) - { - message.AdditionalProperties ??= new(); - message.AdditionalProperties[PersistedMarkerKey] = true; - - foreach (var content in message.Contents) - { - content.AdditionalProperties ??= new(); - content.AdditionalProperties[PersistedMarkerKey] = true; - } - } - } - - /// - /// If the carry the sentinel, - /// returns a clone with the conversation ID cleared so the inner client never sees it. - /// Otherwise returns the original unchanged. - /// - private static ChatOptions? StripLocalHistoryConversationId(ChatOptions? options) - { - if (options?.ConversationId == LocalHistoryConversationId) - { - options = options.Clone(); - options.ConversationId = null; - } - - return options; - } -} diff --git a/dotnet/src/Microsoft.Agents.AI/ChatClient/ServiceStoredSimulatingChatClient.cs b/dotnet/src/Microsoft.Agents.AI/ChatClient/ServiceStoredSimulatingChatClient.cs new file mode 100644 index 0000000000..f7dafa303d --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/ChatClient/ServiceStoredSimulatingChatClient.cs @@ -0,0 +1,273 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI; + +/// +/// A delegating chat client that simulates service-stored chat history behavior using +/// framework-managed instances. +/// +/// +/// +/// This decorator is intended to operate between the and the leaf +/// in a pipeline. +/// +/// +/// Before each service call, it loads chat history from the agent's +/// and prepends it to the request messages. After each successful service call, it persists +/// new request and response messages to the provider. It also returns a sentinel +/// on the response so that the +/// treats the conversation as service-managed — +/// clearing accumulated history between iterations and not injecting duplicate +/// during approval-response processing. +/// +/// +/// This chat client must be used within the context of a running . It retrieves the +/// current agent and session from , which is set automatically when an agent's +/// or +/// +/// method is called. The ensures the run context always contains a resolved session, +/// even when the caller passes null. An is thrown if no run context is +/// available or if the agent is not a . +/// +/// +internal sealed class ServiceStoredSimulatingChatClient : DelegatingChatClient +{ + /// + /// A sentinel value returned on to signal + /// that chat history is being managed downstream. + /// + /// + /// + /// When sees a non-null , + /// it treats the conversation as service-managed: it clears accumulated history between + /// iterations (via FixupHistories) and does not inject + /// into the request during approval-response processing (via ProcessFunctionApprovalResponses). + /// + /// + /// This decorator strips the sentinel from on incoming + /// requests before forwarding to the inner client, so the underlying model never sees it. + /// + /// + internal const string LocalHistoryConversationId = "_agent_local_chat_history"; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying chat client that will handle the core operations. + public ServiceStoredSimulatingChatClient(IChatClient innerClient) + : base(innerClient) + { + } + + /// + public override async Task GetResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + var (agent, session) = GetRequiredAgentAndSession(); + options = StripLocalHistoryConversationId(options); + + bool isServiceManaged = !string.IsNullOrEmpty(options?.ConversationId); + bool isContinuationOrBackground = options?.ContinuationToken is not null + || options?.AllowBackgroundResponses is true; + bool skipSimulation = isServiceManaged || isContinuationOrBackground; + + var newMessages = messages as IList ?? messages.ToList(); + + // When simulating, load history and prepend it. When the service manages + // history (real ConversationId) or this is a continuation/background run, + // just forward the input messages as-is. + var messagesForService = skipSimulation + ? newMessages + : await agent.LoadChatHistoryAsync(session, newMessages, options, cancellationToken).ConfigureAwait(false); + + ChatResponse response; + try + { + response = await base.GetResponseAsync(messagesForService, options, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + await agent.NotifyProvidersOfFailureAsync(session, ex, newMessages, options, cancellationToken).ConfigureAwait(false); + throw; + } + + await agent.NotifyProvidersOfNewMessagesAsync(session, newMessages, response.Messages, options, cancellationToken).ConfigureAwait(false); + + if (isContinuationOrBackground) + { + // Continuation/background run — the agent's forced end-of-run handles + // session ConversationId and persistence; the decorator is a no-op. + } + else if (isServiceManaged || !string.IsNullOrEmpty(response.ConversationId)) + { + // Service manages history — update session with the real ConversationId. + agent.UpdateSessionConversationId(session, response.ConversationId, cancellationToken); + } + else + { + // Normal simulated path — set sentinel so FICC treats this as service-managed. + SetSentinelConversationId(response, session); + } + + return response; + } + + /// + public override async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var (agent, session) = GetRequiredAgentAndSession(); + options = StripLocalHistoryConversationId(options); + + bool isServiceManaged = !string.IsNullOrEmpty(options?.ConversationId); + bool isContinuationOrBackground = options?.ContinuationToken is not null + || options?.AllowBackgroundResponses is true; + bool skipSimulation = isServiceManaged || isContinuationOrBackground; + + var newMessages = messages as IList ?? messages.ToList(); + + // When simulating, load history and prepend it. When the service manages + // history (real ConversationId) or this is a continuation/background run, + // just forward the input messages as-is. + var messagesForService = skipSimulation + ? newMessages + : await agent.LoadChatHistoryAsync(session, newMessages, options, cancellationToken).ConfigureAwait(false); + + List responseUpdates = []; + + IAsyncEnumerator enumerator; + try + { + enumerator = base.GetStreamingResponseAsync(messagesForService, options, cancellationToken).GetAsyncEnumerator(cancellationToken); + } + catch (Exception ex) + { + await agent.NotifyProvidersOfFailureAsync(session, ex, newMessages, options, cancellationToken).ConfigureAwait(false); + throw; + } + + bool hasUpdates; + try + { + hasUpdates = await enumerator.MoveNextAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + await agent.NotifyProvidersOfFailureAsync(session, ex, newMessages, options, cancellationToken).ConfigureAwait(false); + throw; + } + + while (hasUpdates) + { + var update = enumerator.Current; + responseUpdates.Add(update); + + // If the service returned a real ConversationId on any update, remember that. + // Otherwise stamp our sentinel so FICC treats this as service-managed — + // unless this is a continuation/background run where the agent handles everything. + if (!string.IsNullOrEmpty(update.ConversationId)) + { + isServiceManaged = true; + } + else if (!skipSimulation) + { + update.ConversationId = LocalHistoryConversationId; + } + + yield return update; + + try + { + hasUpdates = await enumerator.MoveNextAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + await agent.NotifyProvidersOfFailureAsync(session, ex, newMessages, options, cancellationToken).ConfigureAwait(false); + throw; + } + } + + var chatResponse = responseUpdates.ToChatResponse(); + + await agent.NotifyProvidersOfNewMessagesAsync(session, newMessages, chatResponse.Messages, options, cancellationToken).ConfigureAwait(false); + + if (isContinuationOrBackground) + { + // Continuation/background run — the agent's forced end-of-run handles + // session ConversationId and persistence; the decorator is a no-op. + } + else if (isServiceManaged) + { + // Service manages history — update session with the real ConversationId. + agent.UpdateSessionConversationId(session, chatResponse.ConversationId, cancellationToken); + } + else + { + // Normal simulated path — set sentinel on session. + session.ConversationId = LocalHistoryConversationId; + } + } + + /// + /// Sets the sentinel on the response and session + /// so that treats the conversation as service-managed. + /// + private static void SetSentinelConversationId(ChatResponse response, ChatClientAgentSession session) + { + response.ConversationId = LocalHistoryConversationId; + session.ConversationId = LocalHistoryConversationId; + } + + /// + /// Gets the current and from the run context. + /// + private static (ChatClientAgent Agent, ChatClientAgentSession Session) GetRequiredAgentAndSession() + { + var runContext = AIAgent.CurrentRunContext + ?? throw new InvalidOperationException( + $"{nameof(ServiceStoredSimulatingChatClient)} can only be used within the context of a running AIAgent. " + + "Ensure that the chat client is being invoked as part of an AIAgent.RunAsync or AIAgent.RunStreamingAsync call."); + + var chatClientAgent = runContext.Agent.GetService() + ?? throw new InvalidOperationException( + $"{nameof(ServiceStoredSimulatingChatClient)} can only be used with a {nameof(ChatClientAgent)}. " + + $"The current agent is of type '{runContext.Agent.GetType().Name}'."); + + if (runContext.Session is not ChatClientAgentSession chatClientAgentSession) + { + throw new InvalidOperationException( + $"{nameof(ServiceStoredSimulatingChatClient)} requires a {nameof(ChatClientAgentSession)}. " + + $"The current session is of type '{runContext.Session?.GetType().Name ?? "null"}'."); + } + + return (chatClientAgent, chatClientAgentSession); + } + + /// + /// If the carry the sentinel, + /// returns a clone with the conversation ID cleared so the inner client never sees it. + /// Otherwise returns the original unchanged. + /// + private static ChatOptions? StripLocalHistoryConversationId(ChatOptions? options) + { + if (options?.ConversationId == LocalHistoryConversationId) + { + options = options.Clone(); + options.ConversationId = null; + } + + return options; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/CompatibilitySuppressions.xml b/dotnet/src/Microsoft.Agents.AI/CompatibilitySuppressions.xml new file mode 100644 index 0000000000..941346fbf1 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/CompatibilitySuppressions.xml @@ -0,0 +1,74 @@ + + + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProvider + lib/net10.0/Microsoft.Agents.AI.dll + lib/net10.0/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProviderOptions + lib/net10.0/Microsoft.Agents.AI.dll + lib/net10.0/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProvider + lib/net472/Microsoft.Agents.AI.dll + lib/net472/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProviderOptions + lib/net472/Microsoft.Agents.AI.dll + lib/net472/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProvider + lib/net8.0/Microsoft.Agents.AI.dll + lib/net8.0/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProviderOptions + lib/net8.0/Microsoft.Agents.AI.dll + lib/net8.0/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProvider + lib/net9.0/Microsoft.Agents.AI.dll + lib/net9.0/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProviderOptions + lib/net9.0/Microsoft.Agents.AI.dll + lib/net9.0/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProvider + lib/netstandard2.0/Microsoft.Agents.AI.dll + lib/netstandard2.0/Microsoft.Agents.AI.dll + true + + + CP0001 + T:Microsoft.Agents.AI.FileAgentSkillsProviderOptions + lib/netstandard2.0/Microsoft.Agents.AI.dll + lib/netstandard2.0/Microsoft.Agents.AI.dll + true + + \ No newline at end of file diff --git a/dotnet/src/Microsoft.Agents.AI/Skills/AgentInMemorySkillsSource.cs b/dotnet/src/Microsoft.Agents.AI/Skills/AgentInMemorySkillsSource.cs new file mode 100644 index 0000000000..57c9295c24 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Skills/AgentInMemorySkillsSource.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI; + +/// +/// A skill source that holds instances in memory. +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +internal sealed class AgentInMemorySkillsSource : AgentSkillsSource +{ + private readonly List _skills; + + /// + /// Initializes a new instance of the class. + /// + /// The skills to include in this source. + public AgentInMemorySkillsSource(IEnumerable skills) + { + this._skills = Throw.IfNull(skills).ToList(); + } + + /// + public override Task> GetSkillsAsync(CancellationToken cancellationToken = default) + { + return Task.FromResult>(this._skills); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkill.cs b/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkill.cs index 5f0a66808d..4c5ca8dbc3 100644 --- a/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkill.cs +++ b/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkill.cs @@ -12,7 +12,8 @@ namespace Microsoft.Agents.AI; /// /// /// A skill represents a domain-specific capability with instructions, resources, and scripts. -/// Concrete implementations include (filesystem-backed). +/// Concrete implementations include (filesystem-backed) +/// and (code-defined). /// /// /// Skill metadata follows the Agent Skills specification. @@ -35,6 +36,8 @@ public abstract class AgentSkill /// /// /// For file-based skills this is the raw SKILL.md file content. + /// For code-defined skills this is a synthesized XML document + /// containing name, description, and body (instructions, resources, scripts). /// public abstract string Content { get; } diff --git a/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkillsProvider.cs b/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkillsProvider.cs index f2f87851c0..70d7939227 100644 --- a/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkillsProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkillsProvider.cs @@ -116,6 +116,38 @@ public sealed partial class AgentSkillsProvider : AIContextProvider { } + /// + /// Initializes a new instance of the class + /// with one or more inline (code-defined) skills. + /// Duplicate skill names are automatically deduplicated (first occurrence wins). + /// + /// The inline skills to include. + public AgentSkillsProvider(params AgentInlineSkill[] skills) + : this(skills as IEnumerable) + { + } + + /// + /// Initializes a new instance of the class + /// with inline (code-defined) skills. + /// Duplicate skill names are automatically deduplicated (first occurrence wins). + /// + /// The inline skills to include. + /// Optional provider configuration. + /// Optional logger factory. + public AgentSkillsProvider( + IEnumerable skills, + AgentSkillsProviderOptions? options = null, + ILoggerFactory? loggerFactory = null) + : this( + new DeduplicatingAgentSkillsSource( + new AgentInMemorySkillsSource(Throw.IfNull(skills)), + loggerFactory), + options, + loggerFactory) + { + } + /// /// Initializes a new instance of the class /// from a custom . Unlike other constructors, this one does not diff --git a/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkillsProviderBuilder.cs b/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkillsProviderBuilder.cs index 17d7f2d6f3..8e52cc522e 100644 --- a/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkillsProviderBuilder.cs +++ b/dotnet/src/Microsoft.Agents.AI/Skills/AgentSkillsProviderBuilder.cs @@ -13,9 +13,13 @@ namespace Microsoft.Agents.AI; /// Fluent builder for constructing an backed by a composite source. /// /// +/// +/// Use this builder to combine multiple skill sources into a single provider: +/// /// /// var provider = new AgentSkillsProviderBuilder() /// .UseFileSkills("/path/to/skills") +/// .UseSkills(myInlineSkill1, myInlineSkill2) /// .Build(); /// /// @@ -65,6 +69,40 @@ public sealed class AgentSkillsProviderBuilder return this; } + /// + /// Adds a single skill. + /// + /// The skill to add. + /// This builder instance for chaining. + public AgentSkillsProviderBuilder UseSkill(AgentSkill skill) + { + return this.UseSkills(skill); + } + + /// + /// Adds one or more skills. + /// + /// The skills to add. + /// This builder instance for chaining. + public AgentSkillsProviderBuilder UseSkills(params AgentSkill[] skills) + { + var source = new AgentInMemorySkillsSource(skills); + this._sourceFactories.Add((_, _) => source); + return this; + } + + /// + /// Adds skills from the specified collection. + /// + /// The skills to add. + /// This builder instance for chaining. + public AgentSkillsProviderBuilder UseSkills(IEnumerable skills) + { + var source = new AgentInMemorySkillsSource(skills); + this._sourceFactories.Add((_, _) => source); + return this; + } + /// /// Adds a custom skill source. /// diff --git a/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkill.cs b/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkill.cs new file mode 100644 index 0000000000..d326a47a59 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkill.cs @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI; + +/// +/// A skill defined entirely in code with resources (static values or delegates) and scripts (delegates). +/// +/// +/// All calls to , +/// , and +/// must be made before the skill's is first accessed. +/// Calls made after that point will not be reflected in the generated +/// . In typical usage, this means configuring all +/// resources and scripts before registering the skill with an +/// or . +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class AgentInlineSkill : AgentSkill +{ + private readonly string _instructions; + private List? _resources; + private List? _scripts; + private string? _cachedContent; + + /// + /// Initializes a new instance of the class + /// with a pre-built . + /// + /// The skill frontmatter containing name, description, and other metadata. + /// Skill instructions text. + public AgentInlineSkill(AgentSkillFrontmatter frontmatter, string instructions) + { + this.Frontmatter = Throw.IfNull(frontmatter); + this._instructions = Throw.IfNullOrWhitespace(instructions); + } + + /// + /// Initializes a new instance of the class + /// with all frontmatter properties specified individually. + /// + /// Skill name in kebab-case. + /// Skill description for discovery. + /// Skill instructions text. + /// Optional license name or reference. + /// Optional compatibility information (max 500 chars). + /// Optional space-delimited list of pre-approved tools. + /// Optional arbitrary key-value metadata. + public AgentInlineSkill( + string name, + string description, + string instructions, + string? license = null, + string? compatibility = null, + string? allowedTools = null, + AdditionalPropertiesDictionary? metadata = null) + : this( + new AgentSkillFrontmatter(name, description, compatibility) + { + License = license, + AllowedTools = allowedTools, + Metadata = metadata, + }, + instructions) + { + } + + /// + public override AgentSkillFrontmatter Frontmatter { get; } + + /// + public override string Content => this._cachedContent ??= this.BuildContent(); + + /// + public override IReadOnlyList? Resources => this._resources; + + /// + public override IReadOnlyList? Scripts => this._scripts; + + /// + /// Registers a static resource with this skill. + /// + /// The resource name. + /// The static resource value. + /// An optional description of the resource. + /// This instance, for chaining. + public AgentInlineSkill AddResource(string name, object value, string? description = null) + { + (this._resources ??= []).Add(new AgentInlineSkillResource(name, value, description)); + return this; + } + + /// + /// Registers a dynamic resource with this skill, backed by a C# delegate. + /// The delegate's parameters and return type are automatically marshaled via AIFunctionFactory. + /// + /// The resource name. + /// A method that produces the resource value when requested. + /// An optional description of the resource. + /// This instance, for chaining. + public AgentInlineSkill AddResource(string name, Delegate method, string? description = null) + { + (this._resources ??= []).Add(new AgentInlineSkillResource(name, method, description)); + return this; + } + + /// + /// Registers a script with this skill, backed by a C# delegate. + /// The delegate's parameters and return type are automatically marshaled via AIFunctionFactory. + /// + /// The script name. + /// A method to execute when the script is invoked. + /// An optional description of the script. + /// This instance, for chaining. + public AgentInlineSkill AddScript(string name, Delegate method, string? description = null) + { + (this._scripts ??= []).Add(new AgentInlineSkillScript(name, method, description)); + return this; + } + + private string BuildContent() + { + var sb = new StringBuilder(); + + sb.Append($"{EscapeXmlString(this.Frontmatter.Name)}\n") + .Append($"{EscapeXmlString(this.Frontmatter.Description)}\n\n") + .Append("\n") + .Append(EscapeXmlString(this._instructions)) + .Append("\n"); + + if (this.Resources is { Count: > 0 }) + { + sb.Append("\n\n\n"); + foreach (var resource in this.Resources) + { + if (resource.Description is not null) + { + sb.Append($" \n"); + } + else + { + sb.Append($" \n"); + } + } + + sb.Append(""); + } + + if (this.Scripts is { Count: > 0 }) + { + sb.Append("\n\n\n"); + foreach (var script in this.Scripts) + { + JsonElement? parametersSchema = ((AgentInlineSkillScript)script).ParametersSchema; + + if (script.Description is null && parametersSchema is null) + { + sb.Append($" \n"); + } + } + + sb.Append(""); + } + + return sb.ToString(); + } + + /// + /// Escapes XML special characters: always escapes &, <, >, + /// ", and '. When is , + /// quotes are left unescaped to preserve readability of embedded content such as JSON. + /// + /// The string to escape. + /// + /// When , leaves " and ' unescaped for use in XML element content (e.g., JSON). + /// When (default), escapes all XML special characters including quotes. + /// + private static string EscapeXmlString(string value, bool preserveQuotes = false) + { + var result = value + .Replace("&", "&") + .Replace("<", "<") + .Replace(">", ">"); + + if (!preserveQuotes) + { + result = result + .Replace("\"", """) + .Replace("'", "'"); + } + + return result; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkillResource.cs b/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkillResource.cs new file mode 100644 index 0000000000..38791c3b21 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkillResource.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI; + +/// +/// A skill resource defined in code, backed by either a static value or a delegate. +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +internal sealed class AgentInlineSkillResource : AgentSkillResource +{ + private readonly object? _value; + private readonly AIFunction? _function; + + /// + /// Initializes a new instance of the class with a static value. + /// The value is returned as-is when is called. + /// + /// The resource name. + /// The static resource value. + /// An optional description of the resource. + public AgentInlineSkillResource(string name, object value, string? description = null) + : base(name, description) + { + this._value = Throw.IfNull(value); + } + + /// + /// Initializes a new instance of the class with a delegate. + /// The delegate is invoked via an each time is called, + /// producing a dynamic (computed) value. + /// + /// The resource name. + /// A method that produces the resource value when requested. + /// An optional description of the resource. + public AgentInlineSkillResource(string name, Delegate method, string? description = null) + : base(name, description) + { + Throw.IfNull(method); + this._function = AIFunctionFactory.Create(method, name: this.Name); + } + + /// + public override async Task ReadAsync(IServiceProvider? serviceProvider = null, CancellationToken cancellationToken = default) + { + if (this._function is not null) + { + return await this._function.InvokeAsync(new AIFunctionArguments() { Services = serviceProvider }, cancellationToken).ConfigureAwait(false); + } + + return this._value; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkillScript.cs b/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkillScript.cs new file mode 100644 index 0000000000..acb4f4780b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Skills/Programmatic/AgentInlineSkillScript.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI; + +/// +/// A skill script backed by a delegate. +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +internal sealed class AgentInlineSkillScript : AgentSkillScript +{ + private readonly AIFunction _function; + + /// + /// Initializes a new instance of the class from a delegate. + /// The delegate's parameters and return type are automatically marshaled via . + /// + /// The script name. + /// A method to execute when the script is invoked. Parameters are automatically deserialized from JSON. + /// An optional description of the script. + public AgentInlineSkillScript(string name, Delegate method, string? description = null) + : base(Throw.IfNullOrWhitespace(name), description) + { + Throw.IfNull(method); + this._function = AIFunctionFactory.Create(method, name: this.Name); + } + + /// + /// Gets the JSON schema describing the parameters accepted by this script, or if not available. + /// + public JsonElement? ParametersSchema => this._function.JsonSchema; + + /// + public override async Task RunAsync(AgentSkill skill, AIFunctionArguments arguments, CancellationToken cancellationToken = default) + { + return await this._function.InvokeAsync(arguments, cancellationToken).ConfigureAwait(false); + } +} diff --git a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunStreamingTests.cs b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunStreamingTests.cs index 870dda648c..0f2b123fd9 100644 --- a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunStreamingTests.cs +++ b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunStreamingTests.cs @@ -7,6 +7,8 @@ using Microsoft.Agents.AI; namespace AzureAI.IntegrationTests; +#pragma warning disable CS0618 // Tests intentionally exercise obsolete AIProjectClientFixture +[Obsolete("Use FoundryVersionedAgentRunTests instead. These tests exercise obsolete AIProjectClient extension methods.")] public class AIProjectClientAgentRunStreamingPreviousResponseTests() : RunStreamingTests(() => new()) { public override Task RunWithNoMessageDoesNotFailAsync() @@ -16,7 +18,8 @@ public class AIProjectClientAgentRunStreamingPreviousResponseTests() : RunStream } } -public class AIProjectClientAgentRunStreamingConversationTests() : RunTests(() => new()) +[Obsolete("Use FoundryVersionedAgentRunTests instead. These tests exercise obsolete AIProjectClient extension methods.")] +public class AIProjectClientAgentRunStreamingConversationTests() : RunStreamingTests(() => new()) { public override Func> AgentRunOptionsFactory => async () => { diff --git a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunTests.cs b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunTests.cs index af4cee82e6..ad10ec5b7a 100644 --- a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunTests.cs +++ b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentRunTests.cs @@ -7,6 +7,8 @@ using Microsoft.Agents.AI; namespace AzureAI.IntegrationTests; +#pragma warning disable CS0618 // Tests intentionally exercise obsolete AIProjectClientFixture +[Obsolete("Use FoundryVersionedAgentRunTests instead. These tests exercise obsolete AIProjectClient extension methods.")] public class AIProjectClientAgentRunPreviousResponseTests() : RunTests(() => new()) { public override Task RunWithNoMessageDoesNotFailAsync() @@ -16,6 +18,7 @@ public class AIProjectClientAgentRunPreviousResponseTests() : RunTests(() => new()) { public override Func> AgentRunOptionsFactory => async () => diff --git a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentStructuredOutputRunTests.cs b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentStructuredOutputRunTests.cs index 9db48f3832..b3782a6601 100644 --- a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentStructuredOutputRunTests.cs +++ b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientAgentStructuredOutputRunTests.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Threading.Tasks; using AgentConformance.IntegrationTests; using AgentConformance.IntegrationTests.Support; @@ -8,6 +9,8 @@ using Microsoft.Extensions.AI; namespace AzureAI.IntegrationTests; +#pragma warning disable CS0618 // Tests intentionally exercise obsolete AIProjectClientFixture +[Obsolete("Use FoundryVersionedAgentStructuredOutputRunTests instead. These tests exercise obsolete AIProjectClient extension methods.")] public class AIProjectClientAgentStructuredOutputRunTests() : StructuredOutputRunTests>(() => new AIProjectClientStructuredOutputFixture()) { private const string NotSupported = "AIProjectClient does not support specifying structured output type at invocation time."; @@ -87,6 +90,7 @@ public class AIProjectClientAgentStructuredOutputRunTests() : StructuredOutputRu /// /// Represents a fixture for testing AIProjectClient with structured output of type provided at agent initialization. /// +[Obsolete("Use FoundryVersionedAgentStructuredOutputFixture instead.")] public class AIProjectClientStructuredOutputFixture : AIProjectClientFixture { public override async ValueTask InitializeAsync() diff --git a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunStreamingTests.cs b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunStreamingTests.cs index 3b0c1c27b4..0e507e44c1 100644 --- a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunStreamingTests.cs +++ b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunStreamingTests.cs @@ -1,10 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Threading.Tasks; using AgentConformance.IntegrationTests; namespace AzureAI.IntegrationTests; +#pragma warning disable CS0618 // Tests intentionally exercise obsolete AIProjectClientFixture +[Obsolete("Use FoundryVersionedAgentRunTests instead. These tests exercise obsolete AIProjectClient extension methods.")] public class AIProjectClientChatClientAgentRunStreamingTests() : ChatClientAgentRunStreamingTests(() => new()) { public override Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync() diff --git a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunTests.cs b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunTests.cs index 1e47d0a970..a0ee72ebd4 100644 --- a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunTests.cs +++ b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientChatClientAgentRunTests.cs @@ -1,10 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Threading.Tasks; using AgentConformance.IntegrationTests; namespace AzureAI.IntegrationTests; +#pragma warning disable CS0618 // Tests intentionally exercise obsolete AIProjectClientFixture +[Obsolete("Use FoundryVersionedAgentRunTests instead. These tests exercise obsolete AIProjectClient extension methods.")] public class AIProjectClientChatClientAgentRunTests() : ChatClientAgentRunTests(() => new()) { public override Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync() diff --git a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs index 425197853e..bc5f38acf2 100644 --- a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs +++ b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientCreateTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // Tests intentionally exercise obsolete extension methods + using System; using System.IO; using System.Threading.Tasks; @@ -7,6 +9,7 @@ using AgentConformance.IntegrationTests.Support; using Azure.AI.Projects; using Azure.AI.Projects.Agents; using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; using Microsoft.Extensions.AI; using OpenAI.Files; using OpenAI.Responses; @@ -14,6 +17,7 @@ using Shared.IntegrationTests; namespace AzureAI.IntegrationTests; +[Obsolete("Use FoundryVersionedAgentCreateTests instead. These tests exercise obsolete AIProjectClient extension methods.")] public class AIProjectClientCreateTests { private readonly AIProjectClient _client = new(new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), TestAzureCliCredentials.CreateAzureCliCredential()); @@ -51,7 +55,7 @@ public class AIProjectClientCreateTests Assert.NotNull(agent); Assert.Equal(AgentName, agent.Name); Assert.Equal(AgentDescription, agent.Description); - Assert.Equal(AgentInstructions, agent.Instructions); + Assert.Equal(AgentInstructions, agent.GetService()!.Instructions); var agentRecord = await this._client.Agents.GetAgentAsync(agent.Name); Assert.NotNull(agentRecord); @@ -275,7 +279,7 @@ public class AIProjectClientCreateTests try { // Step 2: Wrap the agent version using AsAIAgent extension. - ChatClientAgent agent = this._client.AsAIAgent(agentVersion); + FoundryAgent agent = this._client.AsAIAgent(agentVersion); // Assert the agent was created correctly and retains version metadata. Assert.NotNull(agent); @@ -327,7 +331,7 @@ public class AIProjectClientCreateTests static string GetWeather(string location) => $"The weather in {location} is sunny with a high of 23C."; var weatherFunction = AIFunctionFactory.Create(GetWeather); - ChatClientAgent agent = createMechanism switch + FoundryAgent agent = createMechanism switch { "CreateWithChatClientAgentOptionsAsync" => await this._client.CreateAIAgentAsync( model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), diff --git a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientFixture.cs b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientFixture.cs index 42892b99b3..112c76571b 100644 --- a/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientFixture.cs +++ b/dotnet/tests/AzureAI.IntegrationTests/AIProjectClientFixture.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // Tests intentionally exercise obsolete extension methods + using System; using System.Collections.Generic; using System.Linq; @@ -9,18 +11,20 @@ using AgentConformance.IntegrationTests.Support; using Azure.AI.Extensions.OpenAI; using Azure.AI.Projects; using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; using Microsoft.Extensions.AI; using OpenAI.Responses; using Shared.IntegrationTests; namespace AzureAI.IntegrationTests; +[Obsolete("Use FoundryVersionedAgentFixture instead. These tests exercise obsolete AIProjectClient extension methods.")] public class AIProjectClientFixture : IChatClientAgentFixture { - private ChatClientAgent _agent = null!; + private FoundryAgent _agent = null!; private AIProjectClient _client = null!; - public IChatClient ChatClient => this._agent.ChatClient; + public IChatClient ChatClient => this._agent.GetService()!.ChatClient; public AIAgent Agent => this._agent; @@ -115,14 +119,14 @@ public class AIProjectClientFixture : IChatClientAgentFixture string instructions = "You are a helpful assistant.", IList? aiTools = null) { - return await this._client.CreateAIAgentAsync(GenerateUniqueAgentName(name), model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), instructions: instructions, tools: aiTools); + return (await this._client.CreateAIAgentAsync(GenerateUniqueAgentName(name), model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), instructions: instructions, tools: aiTools)).GetService()!; } public async Task CreateChatClientAgentAsync(ChatClientAgentOptions options) { options.Name ??= GenerateUniqueAgentName("HelpfulAssistant"); - return await this._client.CreateAIAgentAsync(model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), options); + return (await this._client.CreateAIAgentAsync(model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), options)).GetService()!; } public static string GenerateUniqueAgentName(string baseName) => @@ -170,12 +174,13 @@ public class AIProjectClientFixture : IChatClientAgentFixture public virtual async ValueTask InitializeAsync() { this._client = new(new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), TestAzureCliCredentials.CreateAzureCliCredential()); - this._agent = await this.CreateChatClientAgentAsync(); + this._agent = await this._client.CreateAIAgentAsync(GenerateUniqueAgentName("HelpfulAssistant"), model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), instructions: "You are a helpful assistant."); } public async Task InitializeAsync(ChatClientAgentOptions options) { this._client = new(new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), TestAzureCliCredentials.CreateAzureCliCredential()); - this._agent = await this.CreateChatClientAgentAsync(options); + options.Name ??= GenerateUniqueAgentName("HelpfulAssistant"); + this._agent = await this._client.CreateAIAgentAsync(model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), options); } } diff --git a/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentChatClientRunStreamingTests.cs b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentChatClientRunStreamingTests.cs new file mode 100644 index 0000000000..5895ceb8b9 --- /dev/null +++ b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentChatClientRunStreamingTests.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using AgentConformance.IntegrationTests; + +namespace AzureAI.IntegrationTests; + +public class ResponsesAgentChatClientRunStreamingTests() : ChatClientAgentRunStreamingTests(() => new()) +{ + public override Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync() + { + Assert.Skip("No messages is not supported"); + return base.RunWithInstructionsAndNoMessageReturnsExpectedResultAsync(); + } +} diff --git a/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentChatClientRunTests.cs b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentChatClientRunTests.cs new file mode 100644 index 0000000000..d80b25deb2 --- /dev/null +++ b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentChatClientRunTests.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using AgentConformance.IntegrationTests; + +namespace AzureAI.IntegrationTests; + +public class ResponsesAgentChatClientRunTests() : ChatClientAgentRunTests(() => new()) +{ + public override Task RunWithInstructionsAndNoMessageReturnsExpectedResultAsync() + { + Assert.Skip("No messages is not supported"); + return base.RunWithInstructionsAndNoMessageReturnsExpectedResultAsync(); + } +} diff --git a/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentExtensionCreateTests.cs b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentExtensionCreateTests.cs new file mode 100644 index 0000000000..cd8dc6cb03 --- /dev/null +++ b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentExtensionCreateTests.cs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using AgentConformance.IntegrationTests.Support; +using Azure.AI.Projects; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; +using Microsoft.Extensions.AI; +using Shared.IntegrationTests; + +namespace AzureAI.IntegrationTests; + +/// +/// Integration tests for non-versioned creation via extension methods. +/// +public class ResponsesAgentExtensionCreateTests +{ + private static Uri Endpoint => new(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)); + + private static string Model => TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName); + + private readonly AIProjectClient _client = new(Endpoint, TestAzureCliCredentials.CreateAzureCliCredential()); + + [Fact] + public async Task AsAIAgent_WithModelAndInstructions_CreatesChatClientAgentAndRunsAsync() + { + // Arrange + const string AgentName = "ResponsesAgentExtensionSimple"; + const string AgentDescription = "Integration test agent created from AIProjectClient.AsAIAgent(model, instructions)."; + const string VerificationToken = "integration-extension-ok"; + + FoundryAgent agent = this._client.AsAIAgent( + model: Model, + instructions: $"You are a helpful assistant. When asked for verification, reply with exactly '{VerificationToken}'.", + name: AgentName, + description: AgentDescription); + + AgentSession session = await agent.CreateSessionAsync(); + + try + { + // Act + AgentResponse response = await agent.RunAsync("Return the verification token.", session); + + // Assert + Assert.NotNull(agent); + Assert.Equal(AgentName, agent.Name); + Assert.Equal(AgentDescription, agent.Description); + Assert.Same(this._client, agent.GetService()); + Assert.NotNull(agent.GetService()); + Assert.Contains(VerificationToken, response.Text, StringComparison.OrdinalIgnoreCase); + } + finally + { + await DeleteSessionAsync(this._client, session); + } + } + + [Fact] + public async Task AsAIAgent_WithOptions_CreatesChatClientAgentAndRunsAsync() + { + // Arrange + const string VerificationToken = "integration-options-ok"; + ChatClientAgentOptions options = new() + { + Name = "ResponsesAgentExtensionOptions", + Description = "Integration test agent created from AIProjectClient.AsAIAgent(options).", + ChatOptions = new ChatOptions + { + ModelId = Model, + Instructions = $"You are a helpful assistant. When asked for verification, reply with exactly '{VerificationToken}'.", + }, + }; + + FoundryAgent agent = this._client.AsAIAgent(options); + ChatClientAgentSession session = await agent.CreateConversationSessionAsync(); + + try + { + // Act + AgentResponse response = await agent.RunAsync("Return the verification token.", session); + + // Assert + Assert.StartsWith("conv_", session.ConversationId, StringComparison.OrdinalIgnoreCase); + Assert.Equal(options.Name, agent.Name); + Assert.Equal(options.Description, agent.Description); + Assert.Same(this._client, agent.GetService()); + Assert.Contains(VerificationToken, response.Text, StringComparison.OrdinalIgnoreCase); + } + finally + { + await DeleteSessionAsync(this._client, session); + } + } + + private static async Task DeleteSessionAsync(AIProjectClient client, AgentSession session) + { + ChatClientAgentSession typedSession = (ChatClientAgentSession)session; + + if (typedSession.ConversationId?.StartsWith("conv_", StringComparison.OrdinalIgnoreCase) == true) + { + await client.GetProjectOpenAIClient().GetProjectConversationsClient().DeleteConversationAsync(typedSession.ConversationId); + } + else if (typedSession.ConversationId?.StartsWith("resp_", StringComparison.OrdinalIgnoreCase) == true) + { + await DeleteResponseChainAsync(client, typedSession.ConversationId); + } + } + + private static async Task DeleteResponseChainAsync(AIProjectClient client, string lastResponseId) + { + var responsesClient = client.GetProjectOpenAIClient().GetProjectResponsesClient(); + var response = await responsesClient.GetResponseAsync(lastResponseId); + await responsesClient.DeleteResponseAsync(lastResponseId); + + if (response.Value.PreviousResponseId is not null) + { + await DeleteResponseChainAsync(client, response.Value.PreviousResponseId); + } + } +} diff --git a/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentFixture.cs b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentFixture.cs new file mode 100644 index 0000000000..ec678a00d9 --- /dev/null +++ b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentFixture.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AgentConformance.IntegrationTests; +using AgentConformance.IntegrationTests.Support; +using Azure.AI.Extensions.OpenAI; +using Azure.AI.Projects; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.AzureAI; +using Microsoft.Extensions.AI; +using OpenAI.Responses; +using Shared.IntegrationTests; + +namespace AzureAI.IntegrationTests; + +/// +/// Integration test fixture that creates non-versioned Responses agents via the direct AIProjectClient.AsAIAgent(...) path. +/// +public class ResponsesAgentFixture : IChatClientAgentFixture +{ + private FoundryAgent _agent = null!; + private AIProjectClient _client = null!; + + public IChatClient ChatClient => this._agent.GetService()!.ChatClient; + + public AIAgent Agent => this._agent; + + public async Task CreateConversationAsync() + { + var response = await this._client.GetProjectOpenAIClient().GetProjectConversationsClient().CreateProjectConversationAsync(); + return response.Value.Id; + } + + public async Task> GetChatHistoryAsync(AIAgent agent, AgentSession session) + { + ChatClientAgentSession chatClientSession = (ChatClientAgentSession)session; + + if (chatClientSession.ConversationId?.StartsWith("conv_", StringComparison.OrdinalIgnoreCase) == true) + { + return await this.GetChatHistoryFromConversationAsync(chatClientSession.ConversationId); + } + + if (chatClientSession.ConversationId?.StartsWith("resp_", StringComparison.OrdinalIgnoreCase) == true) + { + return await this.GetChatHistoryFromResponsesChainAsync(chatClientSession.ConversationId); + } + + ChatHistoryProvider? chatHistoryProvider = agent.GetService(); + + if (chatHistoryProvider is null) + { + return []; + } + + return (await chatHistoryProvider.InvokingAsync(new(agent, session, []))).ToList(); + } + + private async Task> GetChatHistoryFromResponsesChainAsync(string conversationId) + { + var openAIResponseClient = this._client.GetProjectOpenAIClient().GetProjectResponsesClient(); + var inputItems = await openAIResponseClient.GetResponseInputItemsAsync(conversationId).ToListAsync(); + var response = await openAIResponseClient.GetResponseAsync(conversationId); + ResponseItem responseItem = response.Value.OutputItems.FirstOrDefault()!; + + var previousMessages = inputItems + .Select(ConvertToChatMessage) + .Where(x => x.Text != "You are a helpful assistant.") + .Reverse(); + + ChatMessage responseMessage = ConvertToChatMessage(responseItem); + + return [.. previousMessages, responseMessage]; + } + + private static ChatMessage ConvertToChatMessage(ResponseItem item) + { + if (item is MessageResponseItem messageResponseItem) + { + ChatRole role = messageResponseItem.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant; + return new ChatMessage(role, messageResponseItem.Content.FirstOrDefault()?.Text); + } + + throw new NotSupportedException("This test currently only supports text messages"); + } + + private async Task> GetChatHistoryFromConversationAsync(string conversationId) + { + List messages = []; + await foreach (AgentResponseItem item in this._client.GetProjectOpenAIClient().GetProjectConversationsClient().GetProjectConversationItemsAsync(conversationId, order: "asc")) + { + var openAIItem = item.AsResponseResultItem(); + if (openAIItem is MessageResponseItem messageItem) + { + messages.Add(new ChatMessage + { + Role = new ChatRole(messageItem.Role.ToString()), + Contents = messageItem.Content + .Where(c => c.Kind is ResponseContentPartKind.OutputText or ResponseContentPartKind.InputText) + .Select(c => new TextContent(c.Text)) + .ToList() + }); + } + } + + return messages; + } + + public Task CreateChatClientAgentAsync( + string name = "HelpfulAssistant", + string instructions = "You are a helpful assistant.", + IList? aiTools = null) + { + return Task.FromResult(this._client.AsAIAgent( + model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), + instructions: instructions, + name: name, + tools: aiTools).GetService()!); + } + + public Task CreateChatClientAgentAsync(ChatClientAgentOptions options) + { + return Task.FromResult(this._client.AsAIAgent(options).GetService()!); + } + + // Non-versioned Responses agents have no server-side agent to delete. + public Task DeleteAgentAsync(ChatClientAgent agent) => Task.CompletedTask; + + public async Task DeleteSessionAsync(AgentSession session) + { + ChatClientAgentSession typedSession = (ChatClientAgentSession)session; + + if (typedSession.ConversationId?.StartsWith("conv_", StringComparison.OrdinalIgnoreCase) == true) + { + await this._client.GetProjectOpenAIClient().GetProjectConversationsClient().DeleteConversationAsync(typedSession.ConversationId); + } + else if (typedSession.ConversationId?.StartsWith("resp_", StringComparison.OrdinalIgnoreCase) == true) + { + await this.DeleteResponseChainAsync(typedSession.ConversationId!); + } + } + + private async Task DeleteResponseChainAsync(string lastResponseId) + { + var response = await this._client.GetProjectOpenAIClient().GetProjectResponsesClient().GetResponseAsync(lastResponseId); + await this._client.GetProjectOpenAIClient().GetProjectResponsesClient().DeleteResponseAsync(lastResponseId); + + if (response.Value.PreviousResponseId is not null) + { + await this.DeleteResponseChainAsync(response.Value.PreviousResponseId); + } + } + + // Non-versioned Responses agents have no server-side agent to clean up on dispose. + public ValueTask DisposeAsync() + { + GC.SuppressFinalize(this); + return default; + } + + public virtual ValueTask InitializeAsync() + { + this._client = new AIProjectClient( + new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), + TestAzureCliCredentials.CreateAzureCliCredential()); + + this._agent = this._client.AsAIAgent( + model: TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), + instructions: "You are a helpful assistant.", + name: "HelpfulAssistant"); + + return default; + } + + public ValueTask InitializeAsync(ChatClientAgentOptions options) + { + this._client = new AIProjectClient( + new Uri(TestConfiguration.GetRequiredValue(TestSettings.AzureAIProjectEndpoint)), + TestAzureCliCredentials.CreateAzureCliCredential()); + + this._agent = this._client.AsAIAgent(options); + + return default; + } +} diff --git a/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentRunStreamingTests.cs b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentRunStreamingTests.cs new file mode 100644 index 0000000000..5f21c316c4 --- /dev/null +++ b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentRunStreamingTests.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using AgentConformance.IntegrationTests; +using Microsoft.Agents.AI; + +namespace AzureAI.IntegrationTests; + +public class ResponsesAgentRunStreamingPreviousResponseTests() : RunStreamingTests(() => new()) +{ + public override Task RunWithNoMessageDoesNotFailAsync() + { + Assert.Skip("No messages is not supported"); + return base.RunWithNoMessageDoesNotFailAsync(); + } +} + +public class ResponsesAgentRunStreamingConversationTests() : RunStreamingTests(() => new()) +{ + public override Func> AgentRunOptionsFactory => async () => + { + var conversationId = await this.Fixture.CreateConversationAsync(); + return new ChatClientAgentRunOptions(new() { ConversationId = conversationId }); + }; + + public override Task RunWithNoMessageDoesNotFailAsync() + { + Assert.Skip("No messages is not supported"); + return base.RunWithNoMessageDoesNotFailAsync(); + } +} diff --git a/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentRunTests.cs b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentRunTests.cs new file mode 100644 index 0000000000..71460f7737 --- /dev/null +++ b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentRunTests.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using AgentConformance.IntegrationTests; +using Microsoft.Agents.AI; + +namespace AzureAI.IntegrationTests; + +public class ResponsesAgentRunPreviousResponseTests() : RunTests(() => new()) +{ + public override Task RunWithNoMessageDoesNotFailAsync() + { + Assert.Skip("No messages is not supported"); + return base.RunWithNoMessageDoesNotFailAsync(); + } +} + +public class ResponsesAgentRunConversationTests() : RunTests(() => new()) +{ + public override Func> AgentRunOptionsFactory => async () => + { + var conversationId = await this.Fixture.CreateConversationAsync(); + return new ChatClientAgentRunOptions(new() { ConversationId = conversationId }); + }; + + public override Task RunWithNoMessageDoesNotFailAsync() + { + Assert.Skip("No messages is not supported"); + return base.RunWithNoMessageDoesNotFailAsync(); + } +} diff --git a/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentStructuredOutputRunTests.cs b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentStructuredOutputRunTests.cs new file mode 100644 index 0000000000..19ebdb4e28 --- /dev/null +++ b/dotnet/tests/AzureAI.IntegrationTests/ResponsesAgentStructuredOutputRunTests.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using AgentConformance.IntegrationTests; +using AgentConformance.IntegrationTests.Support; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using Shared.IntegrationTests; + +namespace AzureAI.IntegrationTests; + +public class ResponsesAgentStructuredOutputRunTests() : StructuredOutputRunTests>(() => new()) +{ + private const string NotSupported = "The direct Responses AsAIAgent path does not support specifying structured output type at invocation time."; + + /// + /// Verifies that response format provided at agent initialization is used when invoking RunAsync. + /// + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public async Task RunWithResponseFormatAtAgentInitializationReturnsExpectedResultAsync() + { + // Arrange + AIAgent agent = this.Fixture.Agent; + AgentSession session = await agent.CreateSessionAsync(); + await using var cleanup = new SessionCleanup(session, this.Fixture); + + // Act + AgentResponse response = await agent.RunAsync(new ChatMessage(ChatRole.User, "Provide information about the capital of France."), session); + + // Assert + Assert.NotNull(response); + Assert.Single(response.Messages); + Assert.Contains("Paris", response.Text); + Assert.True(TryDeserialize(response.Text, AgentAbstractionsJsonUtilities.DefaultOptions, out CityInfo cityInfo)); + Assert.Equal("Paris", cityInfo.Name); + } + + /// + /// Verifies that generic RunAsync works when structured output is configured at agent initialization. + /// + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public async Task RunGenericWithResponseFormatAtAgentInitializationReturnsExpectedResultAsync() + { + // Arrange + AIAgent agent = this.Fixture.Agent; + AgentSession session = await agent.CreateSessionAsync(); + await using var cleanup = new SessionCleanup(session, this.Fixture); + + // Act + AgentResponse response = await agent.RunAsync( + new ChatMessage(ChatRole.User, "Provide information about the capital of France."), + session); + + // Assert + Assert.NotNull(response); + Assert.Single(response.Messages); + Assert.Contains("Paris", response.Text); + + Assert.NotNull(response.Result); + Assert.Equal("Paris", response.Result.Name); + } + + public override Task RunWithGenericTypeReturnsExpectedResultAsync() + { + Assert.Skip(NotSupported); + return base.RunWithGenericTypeReturnsExpectedResultAsync(); + } + + public override Task RunWithResponseFormatReturnsExpectedResultAsync() + { + Assert.Skip(NotSupported); + return base.RunWithResponseFormatReturnsExpectedResultAsync(); + } + + public override Task RunWithPrimitiveTypeReturnsExpectedResultAsync() + { + Assert.Skip(NotSupported); + return base.RunWithPrimitiveTypeReturnsExpectedResultAsync(); + } +} + +/// +/// Fixture for testing the direct Responses path with structured output of type provided at agent initialization. +/// +public class ResponsesAgentStructuredOutputFixture : ResponsesAgentFixture +{ + public override ValueTask InitializeAsync() + { + ChatClientAgentOptions agentOptions = new() + { + ChatOptions = new ChatOptions() + { + ModelId = TestConfiguration.GetRequiredValue(TestSettings.AzureAIModelDeploymentName), + ResponseFormat = ChatResponseFormat.ForJsonSchema(AgentAbstractionsJsonUtilities.DefaultOptions) + }, + }; + + return this.InitializeAsync(agentOptions); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIStreamingMessageIdTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIStreamingMessageIdTests.cs new file mode 100644 index 0000000000..5c55408ff8 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIStreamingMessageIdTests.cs @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI.AGUI.Shared; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.AGUI.UnitTests; + +/// +/// Tests for AGUI streaming behavior when MessageId is null or missing from +/// ChatResponseUpdate objects (e.g., providers like Google GenAI/Vertex AI +/// that don't supply MessageId on streaming chunks). +/// +public sealed class AGUIStreamingMessageIdTests +{ + /// + /// When ChatResponseUpdate objects with null MessageId are fed directly to + /// AsAGUIEventStreamAsync, the AGUI layer generates a fallback MessageId so + /// that events are valid regardless of agent type or provider. + /// + [Fact] + public async Task TextStreaming_NullMessageId_GeneratesFallbackInAGUILayerAsync() + { + // Arrange - Simulate a provider that does NOT set MessageId + List providerUpdates = + [ + new ChatResponseUpdate(ChatRole.Assistant, "Hello"), + new ChatResponseUpdate(ChatRole.Assistant, " world"), + new ChatResponseUpdate(ChatRole.Assistant, "!") + ]; + + // Act + List aguiEvents = []; + await foreach (BaseEvent evt in providerUpdates.ToAsyncEnumerableAsync() + .AsAGUIEventStreamAsync("thread-1", "run-1", AGUIJsonSerializerContext.Default.Options)) + { + aguiEvents.Add(evt); + } + + // Assert - AGUI layer should generate a fallback MessageId + List startEvents = aguiEvents.OfType().ToList(); + List contentEvents = aguiEvents.OfType().ToList(); + + Assert.Single(startEvents); + Assert.False(string.IsNullOrEmpty(startEvents[0].MessageId)); + + Assert.Equal(3, contentEvents.Count); + Assert.All(contentEvents, e => Assert.False(string.IsNullOrEmpty(e.MessageId))); + + // All events should share the same generated MessageId + string?[] distinctIds = contentEvents.Select(e => e.MessageId).Distinct().ToArray(); + Assert.Single(distinctIds); + Assert.Equal(startEvents[0].MessageId, distinctIds[0]); + } + + /// + /// Full pipeline: ChatClientAgent → AsChatResponseUpdatesAsync → AsAGUIEventStreamAsync + /// with a provider that returns null MessageId. Verifies that fallback MessageId + /// generation ensures valid AGUI events. + /// + [Fact] + public async Task FullPipeline_NullProviderMessageId_ProducesValidAGUIEventsAsync() + { + // Arrange - ChatClientAgent with a mock client that omits MessageId + IChatClient mockChatClient = new NullMessageIdChatClient(); + ChatClientAgent agent = new(mockChatClient, name: "test-agent"); + + ChatMessage userMessage = new(ChatRole.User, "tell me about agents"); + + // Act - Run the full pipeline exactly as MapAGUI does + List aguiEvents = []; + await foreach (BaseEvent evt in agent + .RunStreamingAsync([userMessage]) + .AsChatResponseUpdatesAsync() + .AsAGUIEventStreamAsync("thread-1", "run-1", AGUIJsonSerializerContext.Default.Options)) + { + aguiEvents.Add(evt); + } + + // Assert — The pipeline should produce AGUI events with valid messageId + List startEvents = aguiEvents.OfType().ToList(); + List contentEvents = aguiEvents.OfType().ToList(); + + Assert.NotEmpty(startEvents); + Assert.NotEmpty(contentEvents); + + foreach (TextMessageStartEvent startEvent in startEvents) + { + Assert.False( + string.IsNullOrEmpty(startEvent.MessageId), + "TextMessageStartEvent.MessageId should not be null/empty when provider omits it"); + } + + foreach (TextMessageContentEvent contentEvent in contentEvents) + { + Assert.False( + string.IsNullOrEmpty(contentEvent.MessageId), + "TextMessageContentEvent.MessageId should not be null/empty when provider omits it"); + } + + // All content events should share the same messageId + string?[] distinctMessageIds = contentEvents.Select(e => e.MessageId).Distinct().ToArray(); + Assert.Single(distinctMessageIds); + } + + /// + /// When ChatResponseUpdate has empty string MessageId, the AGUI layer generates + /// a fallback so ToolCallStartEvent.ParentMessageId is valid. + /// + [Fact] + public async Task ToolCalls_EmptyMessageId_GeneratesFallbackParentMessageIdAsync() + { + // Arrange - ChatResponseUpdate with a tool call but empty MessageId + FunctionCallContent functionCall = new("call_abc123", "GetWeather") + { + Arguments = new Dictionary { ["location"] = "San Francisco" } + }; + + List providerUpdates = + [ + new ChatResponseUpdate + { + Role = ChatRole.Assistant, + MessageId = "", + Contents = [functionCall] + } + ]; + + // Act + List aguiEvents = []; + await foreach (BaseEvent evt in providerUpdates.ToAsyncEnumerableAsync() + .AsAGUIEventStreamAsync("thread-1", "run-1", AGUIJsonSerializerContext.Default.Options)) + { + aguiEvents.Add(evt); + } + + // Assert — ParentMessageId should have a generated fallback + ToolCallStartEvent? toolCallStart = aguiEvents.OfType().FirstOrDefault(); + Assert.NotNull(toolCallStart); + Assert.Equal("call_abc123", toolCallStart.ToolCallId); + Assert.Equal("GetWeather", toolCallStart.ToolCallName); + Assert.False( + string.IsNullOrEmpty(toolCallStart.ParentMessageId), + "ParentMessageId should have a generated fallback for empty provider MessageId"); + } + + /// + /// When a provider properly sets MessageId (e.g., OpenAI), the AGUI pipeline + /// produces valid events with correct messageId values. + /// + [Fact] + public async Task TextStreaming_WithProviderMessageId_ProducesValidAGUIEventsAsync() + { + // Arrange — Provider that properly sets MessageId + List providerUpdates = + [ + new ChatResponseUpdate(ChatRole.Assistant, "Hello") + { + MessageId = "chatcmpl-abc123" + }, + new ChatResponseUpdate(ChatRole.Assistant, " world") + { + MessageId = "chatcmpl-abc123" + } + ]; + + // Act + List aguiEvents = []; + await foreach (BaseEvent evt in providerUpdates.ToAsyncEnumerableAsync() + .AsAGUIEventStreamAsync("thread-1", "run-1", AGUIJsonSerializerContext.Default.Options)) + { + aguiEvents.Add(evt); + } + + // Assert + List startEvents = aguiEvents.OfType().ToList(); + List contentEvents = aguiEvents.OfType().ToList(); + + Assert.Single(startEvents); + Assert.Equal("chatcmpl-abc123", startEvents[0].MessageId); + + Assert.Equal(2, contentEvents.Count); + Assert.All(contentEvents, e => Assert.Equal("chatcmpl-abc123", e.MessageId)); + } +} + +/// +/// Mock IChatClient that simulates a provider not setting MessageId on streaming chunks +/// (e.g., Google GenAI / Vertex AI). +/// +internal sealed class NullMessageIdChatClient : IChatClient +{ + public void Dispose() + { + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public Task GetResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + { + return Task.FromResult(new ChatResponse([new(ChatRole.Assistant, "response")])); + } + + public async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + foreach (string chunk in (string[])["Agents", " are", " autonomous", " programs."]) + { + yield return new ChatResponseUpdate + { + Role = ChatRole.Assistant, + Contents = [new TextContent(chunk)] + }; + + await Task.Yield(); + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/AgentResponseUpdateExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/AgentResponseUpdateExtensionsTests.cs index 89cff04de8..5cf0b75eef 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/AgentResponseUpdateExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/AgentResponseUpdateExtensionsTests.cs @@ -384,6 +384,45 @@ public class AgentResponseUpdateExtensionsTests Assert.Same(originalChatResponseUpdate, result); } + [Fact] + public void AsChatResponseUpdate_WithRawRepresentationNullMessageId_ReturnsRawDirectly() + { + // Arrange - RawRepresentation has null MessageId + ChatResponseUpdate originalChatResponseUpdate = new() + { + ResponseId = "original-update", + Contents = [new TextContent("Hello")] + }; + AgentResponseUpdate agentResponseUpdate = new(originalChatResponseUpdate); + + // Act + ChatResponseUpdate result = agentResponseUpdate.AsChatResponseUpdate(); + + // Assert - Returns the raw representation directly without mutation + Assert.Same(originalChatResponseUpdate, result); + Assert.Null(result.MessageId); + } + + [Fact] + public void AsChatResponseUpdate_WithRawRepresentationExistingMessageId_PreservesOriginal() + { + // Arrange - RawRepresentation already has MessageId set by provider + ChatResponseUpdate originalChatResponseUpdate = new() + { + ResponseId = "original-update", + MessageId = "provider-message-id", + Contents = [new TextContent("Hello")] + }; + AgentResponseUpdate agentResponseUpdate = new(originalChatResponseUpdate); + + // Act + ChatResponseUpdate result = agentResponseUpdate.AsChatResponseUpdate(); + + // Assert - Provider's original MessageId should be preserved + Assert.Same(originalChatResponseUpdate, result); + Assert.Equal("provider-message-id", result.MessageId); + } + [Fact] public void AsChatResponseUpdate_WithoutRawRepresentation_CreatesNewChatResponseUpdate() { diff --git a/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/AzureAIProjectChatClientExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/AzureAIProjectChatClientExtensionsTests.cs index 261faaded8..5c954d30e8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/AzureAIProjectChatClientExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/AzureAIProjectChatClientExtensionsTests.cs @@ -21,11 +21,173 @@ using OpenAI.Responses; namespace Microsoft.Agents.AI.AzureAI.UnitTests; +#pragma warning disable CS0618 /// /// Unit tests for the class. /// +[Obsolete("Includes coverage for obsolete AIProjectClient compatibility extension methods.")] public sealed class AzureAIProjectChatClientExtensionsTests { + #region AsAIAgent(AIProjectClient, model, instructions) Tests + + /// + /// Verify that the non-versioned AsAIAgent overload throws ArgumentNullException when AIProjectClient is null. + /// + [Fact] + public void AsAIAgent_WithModelAndInstructions_WithNullClient_ThrowsArgumentNullException() + { + // Arrange + AIProjectClient? client = null; + + // Act & Assert + ArgumentNullException exception = Assert.Throws(() => + client!.AsAIAgent("gpt-4o-mini", "You are helpful.")); + + Assert.Equal("aiProjectClient", exception.ParamName); + } + + /// + /// Verify that the non-versioned AsAIAgent overload creates a valid ChatClientAgent. + /// + [Fact] + public void AsAIAgent_WithModelAndInstructions_CreatesChatClientAgent() + { + // Arrange + AIProjectClient client = this.CreateTestAgentClient(); + List tools = + [ + AIFunctionFactory.Create(() => "test", "test_function", "A test function") + ]; + + // Act + FoundryAgent agent = client.AsAIAgent( + "gpt-4o-mini", + "You are helpful.", + name: "test-agent", + description: "A test agent", + tools: tools); + + // Assert + Assert.NotNull(agent); + Assert.Equal("test-agent", agent.Name); + Assert.Equal("A test agent", agent.Description); + Assert.Same(client, agent.GetService()); + Assert.NotNull(agent.GetService()); + } + + /// + /// Verify that the non-versioned AsAIAgent overload applies the clientFactory. + /// + [Fact] + public void AsAIAgent_WithModelAndInstructions_WithClientFactory_AppliesFactoryCorrectly() + { + // Arrange + AIProjectClient client = this.CreateTestAgentClient(); + TestChatClient? testChatClient = null; + + // Act + FoundryAgent agent = client.AsAIAgent( + "gpt-4o-mini", + "You are helpful.", + clientFactory: innerClient => testChatClient = new TestChatClient(innerClient)); + + // Assert + Assert.NotNull(agent); + TestChatClient? retrievedTestClient = agent.GetService(); + Assert.NotNull(retrievedTestClient); + Assert.Same(testChatClient, retrievedTestClient); + } + + /// + /// Verify that the options-based non-versioned AsAIAgent overload creates a valid ChatClientAgent. + /// + [Fact] + public void AsAIAgent_WithOptions_CreatesChatClientAgent() + { + // Arrange + AIProjectClient client = this.CreateTestAgentClient(); + ChatClientAgentOptions options = new() + { + Name = "options-agent", + Description = "Agent from options", + ChatOptions = new ChatOptions + { + ModelId = "gpt-4o-mini", + Instructions = "You are helpful.", + }, + }; + + // Act + FoundryAgent agent = client.AsAIAgent(options); + + // Assert + Assert.NotNull(agent); + Assert.Equal("options-agent", agent.Name); + Assert.Equal("Agent from options", agent.Description); + Assert.Same(client, agent.GetService()); + } + + /// + /// Verify that the non-versioned AsAIAgent overload adds the MEAI user-agent header to Responses API requests. + /// + [Fact] + public async Task AsAIAgent_WithModelAndInstructions_UserAgentHeaderAddedToResponsesRequestsAsync() + { + // Arrange + bool userAgentFound = false; + using HttpHandlerAssert httpHandler = new(request => + { + if (request.Headers.TryGetValues("User-Agent", out IEnumerable? values)) + { + foreach (string value in values) + { + if (value.Contains("MEAI")) + { + userAgentFound = true; + } + } + } + + if (request.Method == HttpMethod.Post && request.RequestUri!.PathAndQuery.Contains("/responses")) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + TestDataUtil.GetOpenAIDefaultResponseJson(), + Encoding.UTF8, + "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}", Encoding.UTF8, "application/json") + }; + }); + +#pragma warning disable CA5399 + using HttpClient httpClient = new(httpHandler); +#pragma warning restore CA5399 + + AIProjectClient aiProjectClient = new( + new Uri("https://test.openai.azure.com/"), + new FakeAuthenticationTokenProvider(), + new() { Transport = new HttpClientPipelineTransport(httpClient) }); + + FoundryAgent agent = aiProjectClient.AsAIAgent( + "gpt-4o-mini", + "You are helpful."); + + // Act + AgentSession session = await agent.CreateSessionAsync(); + await agent.RunAsync("Hello", session); + + // Assert + Assert.True(userAgentFound, "MEAI user-agent header was not found in any request"); + } + + #endregion + #region AsAIAgent(AIProjectClient, AgentRecord) Tests /// @@ -199,7 +361,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -217,7 +379,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } #endregion @@ -409,7 +571,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); var chatClient = agent.GetService(); Assert.NotNull(chatClient); var agentVersion = chatClient.GetService(); @@ -458,7 +620,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -481,7 +643,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); Assert.Equal("test-agent", agent.Name); - Assert.Equal("Test instructions", agent.Instructions); + Assert.Equal("Test instructions", agent.GetService()!.Instructions); } /// @@ -570,7 +732,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -592,7 +754,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -612,7 +774,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -638,7 +800,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); var agentVersion = agent.GetService(); Assert.NotNull(agentVersion); if (agentVersion.Definition is PromptAgentDefinition promptDef) @@ -677,7 +839,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); var agentVersion = agent.GetService(); Assert.NotNull(agentVersion); if (agentVersion.Definition is PromptAgentDefinition promptDef) @@ -721,7 +883,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); var agentVersion = agent.GetService(); Assert.NotNull(agentVersion); Assert.IsType(agentVersion.Definition); @@ -783,7 +945,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); var chatClient = agent.GetService(); Assert.NotNull(chatClient); var agentVersion = chatClient.GetService(); @@ -815,7 +977,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); var agentVersion = agent.GetService(); Assert.NotNull(agentVersion); if (agentVersion.Definition is PromptAgentDefinition promptDef) @@ -843,7 +1005,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -864,7 +1026,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } #endregion @@ -912,7 +1074,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); var agentVersion = agent.GetService(); Assert.NotNull(agentVersion); if (agentVersion.Definition is PromptAgentDefinition promptDef) @@ -952,7 +1114,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -979,7 +1141,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -1011,7 +1173,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } #endregion @@ -1064,7 +1226,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); Assert.Equal("test-agent", agent.Name); - Assert.Equal("Custom instructions", agent.Instructions); + Assert.Equal("Custom instructions", agent.GetService()!.Instructions); Assert.Equal("Custom description", agent.Description); } @@ -1353,7 +1515,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests // Assert Assert.NotNull(agent); Assert.Equal(AgentName, agent.Name); - Assert.Equal(Instructions, agent.Instructions); + Assert.Equal(Instructions, agent.GetService()!.Instructions); var wrappedClient = agent.GetService(); Assert.NotNull(wrappedClient); } @@ -1957,11 +2119,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -1983,11 +2145,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2011,11 +2173,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2044,11 +2206,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2077,11 +2239,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } #endregion @@ -2107,11 +2269,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2133,11 +2295,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2159,11 +2321,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } #endregion @@ -2186,7 +2348,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); @@ -2208,7 +2370,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); @@ -2303,11 +2465,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } #endregion @@ -2330,7 +2492,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); @@ -2352,7 +2514,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); @@ -2374,7 +2536,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); @@ -2400,7 +2562,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - should not throw even without tools when UseProvidedChatClientAsIs is true - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); @@ -2432,7 +2594,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - UseProvidedChatClientAsIs is true, but provided AIFunctions should still be matched and preserved - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); @@ -2463,11 +2625,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); // Verify the agent ID is generated from server-returned name ("agent_abc123") and "latest" Assert.Equal("agent_abc123:latest", agent.Id); } @@ -2525,11 +2687,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); // Verify the agent ID is generated from server-returned name ("agent_abc123") and "latest" Assert.Equal("agent_abc123:latest", agent.Id); } @@ -2679,11 +2841,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2707,11 +2869,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); + FoundryAgent agent = await testClient.Client.CreateAIAgentAsync("test-model", options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2745,11 +2907,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2783,11 +2945,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests }; // Act - ChatClientAgent agent = await client.GetAIAgentAsync(options); + FoundryAgent agent = await client.GetAIAgentAsync(options); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } /// @@ -2841,11 +3003,11 @@ public sealed class AzureAIProjectChatClientExtensionsTests AgentVersion agentVersion = ModelReaderWriter.Read(BinaryData.FromString(TestDataUtil.GetAgentVersionResponseJson(agentDefinition: definition)))!; // Act - no tools provided, but requireInvocableTools is false when no tools param is passed - ChatClientAgent agent = client.AsAIAgent(agentVersion); + FoundryAgent agent = client.AsAIAgent(agentVersion); // Assert Assert.NotNull(agent); - Assert.IsType(agent); + Assert.IsType(agent); } #endregion @@ -3135,17 +3297,16 @@ public sealed class AzureAIProjectChatClientExtensionsTests /// private sealed class MockPipelineResponse : PipelineResponse { - private readonly int _status; private readonly MockPipelineResponseHeaders _headers; public MockPipelineResponse(int status, BinaryData? content = null) { - this._status = status; + this.Status = status; this.Content = content ?? BinaryData.Empty; this._headers = new MockPipelineResponseHeaders(); } - public override int Status => this._status; + public override int Status { get; } public override string ReasonPhrase => "OK"; @@ -3206,9 +3367,10 @@ public sealed class AzureAIProjectChatClientExtensionsTests /// /// Helper method to access internal ChatOptions property via reflection. /// - private static ChatOptions? GetAgentChatOptions(ChatClientAgent agent) + private static ChatOptions? GetAgentChatOptions(AIAgent agent) { - if (agent is null) + ChatClientAgent? chatClientAgent = agent as ChatClientAgent ?? agent.GetService(); + if (chatClientAgent is null) { return null; } @@ -3219,7 +3381,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); - return chatOptionsProperty?.GetValue(agent) as ChatOptions; + return chatOptionsProperty?.GetValue(chatClientAgent) as ChatOptions; } /// @@ -3260,6 +3422,7 @@ public sealed class AzureAIProjectChatClientExtensionsTests } } } +#pragma warning restore CS0618 /// /// Provides test data for invalid agent name validation tests. diff --git a/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/AzureAIProjectChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/AzureAIProjectChatClientTests.cs index 5c61e0b457..8582ccc2b6 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/AzureAIProjectChatClientTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/AzureAIProjectChatClientTests.cs @@ -10,6 +10,8 @@ using Azure.AI.Projects; namespace Microsoft.Agents.AI.AzureAI.UnitTests; +#pragma warning disable CS0618 +[Obsolete("Uses obsolete AIProjectClient.GetAIAgentAsync compatibility extensions while validating chat-client behavior.")] public class AzureAIProjectChatClientTests { /// @@ -43,9 +45,12 @@ public class AzureAIProjectChatClientTests using var httpClient = new HttpClient(httpHandler); #pragma warning restore CA5399 - var client = new AIProjectClient(new Uri("https://test.openai.azure.com/"), new FakeAuthenticationTokenProvider(), new() { Transport = new HttpClientPipelineTransport(httpClient) }); + AIProjectClient projectClient = new( + new Uri("https://test.openai.azure.com/"), + new FakeAuthenticationTokenProvider(), + new AIProjectClientOptions() { Transport = new HttpClientPipelineTransport(httpClient) }); - var agent = await client.GetAIAgentAsync( + var agent = await projectClient.GetAIAgentAsync( new ChatClientAgentOptions { Name = "test-agent", @@ -92,9 +97,12 @@ public class AzureAIProjectChatClientTests using var httpClient = new HttpClient(httpHandler); #pragma warning restore CA5399 - var client = new AIProjectClient(new Uri("https://test.openai.azure.com/"), new FakeAuthenticationTokenProvider(), new() { Transport = new HttpClientPipelineTransport(httpClient) }); + AIProjectClient projectClient = new( + new Uri("https://test.openai.azure.com/"), + new FakeAuthenticationTokenProvider(), + new AIProjectClientOptions() { Transport = new HttpClientPipelineTransport(httpClient) }); - var agent = await client.GetAIAgentAsync( + var agent = await projectClient.GetAIAgentAsync( new ChatClientAgentOptions { Name = "test-agent", @@ -141,9 +149,12 @@ public class AzureAIProjectChatClientTests using var httpClient = new HttpClient(httpHandler); #pragma warning restore CA5399 - var client = new AIProjectClient(new Uri("https://test.openai.azure.com/"), new FakeAuthenticationTokenProvider(), new() { Transport = new HttpClientPipelineTransport(httpClient) }); + AIProjectClient projectClient = new( + new Uri("https://test.openai.azure.com/"), + new FakeAuthenticationTokenProvider(), + new AIProjectClientOptions() { Transport = new HttpClientPipelineTransport(httpClient) }); - var agent = await client.GetAIAgentAsync( + var agent = await projectClient.GetAIAgentAsync( new ChatClientAgentOptions { Name = "test-agent", @@ -190,9 +201,12 @@ public class AzureAIProjectChatClientTests using var httpClient = new HttpClient(httpHandler); #pragma warning restore CA5399 - var client = new AIProjectClient(new Uri("https://test.openai.azure.com/"), new FakeAuthenticationTokenProvider(), new() { Transport = new HttpClientPipelineTransport(httpClient) }); + AIProjectClient projectClient = new( + new Uri("https://test.openai.azure.com/"), + new FakeAuthenticationTokenProvider(), + new AIProjectClientOptions() { Transport = new HttpClientPipelineTransport(httpClient) }); - var agent = await client.GetAIAgentAsync( + var agent = await projectClient.GetAIAgentAsync( new ChatClientAgentOptions { Name = "test-agent", @@ -208,3 +222,4 @@ public class AzureAIProjectChatClientTests Assert.Equal("resp_0888a46cbf2b1ff3006914596e05d08195a77c3f5187b769a7", chatClientSession.ConversationId); } } +#pragma warning restore CS0618 diff --git a/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/FoundryAgentTests.cs b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/FoundryAgentTests.cs new file mode 100644 index 0000000000..1b77e8f57e --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/FoundryAgentTests.cs @@ -0,0 +1,316 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Azure.AI.Projects; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.AzureAI.UnitTests; + +/// +/// Unit tests for the class. +/// +public class FoundryAgentTests +{ + private static readonly Uri s_testEndpoint = new("https://test.services.ai.azure.com/api/projects/test-project"); + + #region Constructor validation tests + + [Fact] + public void Constructor_WithNullEndpoint_ThrowsArgumentNullException() + { + ArgumentNullException exception = Assert.Throws(() => + new FoundryAgent( + projectEndpoint: null!, + credential: new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test instructions")); + + Assert.Equal("endpoint", exception.ParamName); + } + + [Fact] + public void Constructor_WithNullCredential_ThrowsArgumentNullException() + { + ArgumentNullException exception = Assert.Throws(() => + new FoundryAgent( + projectEndpoint: s_testEndpoint, + credential: null!, + model: "gpt-4o-mini", + instructions: "Test instructions")); + + Assert.Equal("credential", exception.ParamName); + } + + [Fact] + public void Constructor_WithNullModel_ThrowsArgumentException() + { + Assert.ThrowsAny(() => + new FoundryAgent( + projectEndpoint: s_testEndpoint, + credential: new FakeAuthenticationTokenProvider(), + model: null!, + instructions: "Test instructions")); + } + + [Fact] + public void Constructor_WithEmptyModel_ThrowsArgumentException() + { + Assert.ThrowsAny(() => + new FoundryAgent( + projectEndpoint: s_testEndpoint, + credential: new FakeAuthenticationTokenProvider(), + model: string.Empty, + instructions: "Test instructions")); + } + + [Fact] + public void Constructor_WithNullInstructions_ThrowsArgumentException() + { + Assert.ThrowsAny(() => + new FoundryAgent( + projectEndpoint: s_testEndpoint, + credential: new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: null!)); + } + + [Fact] + public void Constructor_WithValidParams_CreatesAgent() + { + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "You are a helpful assistant.", + name: "test-agent", + description: "A test agent"); + + Assert.NotNull(agent); + Assert.Equal("test-agent", agent.Name); + Assert.Equal("A test agent", agent.Description); + } + + #endregion + + #region Property tests + + [Fact] + public void Name_ReturnsConfiguredName() + { + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test", + name: "my-agent"); + + Assert.Equal("my-agent", agent.Name); + } + + [Fact] + public void Description_ReturnsConfiguredDescription() + { + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test", + description: "Agent description"); + + Assert.Equal("Agent description", agent.Description); + } + + [Fact] + public void GetService_ReturnsAIProjectClient() + { + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test"); + + AIProjectClient? client = agent.GetService(); + + Assert.NotNull(client); + } + + [Fact] + public void GetService_ReturnsChatClientAgent() + { + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test"); + + ChatClientAgent? innerAgent = agent.GetService(); + + Assert.NotNull(innerAgent); + } + + [Fact] + public void GetService_ReturnsIChatClient() + { + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test"); + + IChatClient? chatClient = agent.GetService(); + + Assert.NotNull(chatClient); + } + + [Fact] + public void GetService_ReturnsChatClientMetadata() + { + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test"); + + ChatClientMetadata? metadata = agent.GetService(); + + Assert.NotNull(metadata); + Assert.Equal("microsoft.foundry", metadata.ProviderName); + } + + [Fact] + public void GetService_ReturnsNullForUnknownType() + { + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test"); + + Assert.Null(agent.GetService()); + } + + #endregion + + #region Functional tests + + [Fact] + public async Task RunAsync_SendsRequestToResponsesAPIAsync() + { + bool requestTriggered = false; + using HttpHandlerAssert httpHandler = new(request => + { + if (request.Method == HttpMethod.Post && request.RequestUri!.PathAndQuery.Contains("/responses")) + { + requestTriggered = true; + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + TestDataUtil.GetOpenAIDefaultResponseJson(), + Encoding.UTF8, + "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}", Encoding.UTF8, "application/json") + }; + }); + +#pragma warning disable CA5399 + using HttpClient httpClient = new(httpHandler); +#pragma warning restore CA5399 + + AIProjectClientOptions clientOptions = new() + { + Transport = new HttpClientPipelineTransport(httpClient) + }; + + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "You are a helpful assistant.", + clientOptions: clientOptions); + + AgentSession session = await agent.CreateSessionAsync(); + await agent.RunAsync("Hello", session); + + Assert.True(requestTriggered); + } + + [Fact] + public void Constructor_WithChatClientFactory_AppliesFactory() + { + bool factoryCalled = false; + + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test", + clientFactory: client => + { + factoryCalled = true; + return client; + }); + + Assert.True(factoryCalled); + Assert.NotNull(agent); + } + + [Fact] + public async Task Constructor_UserAgentHeaderAddedToRequestsAsync() + { + bool userAgentFound = false; + using HttpHandlerAssert httpHandler = new(request => + { + if (request.Headers.TryGetValues("User-Agent", out System.Collections.Generic.IEnumerable? values)) + { + foreach (string value in values) + { + if (value.StartsWith("MEAI/", StringComparison.OrdinalIgnoreCase)) + { + userAgentFound = true; + } + } + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + TestDataUtil.GetOpenAIDefaultResponseJson(), + Encoding.UTF8, + "application/json") + }; + }); + +#pragma warning disable CA5399 + using HttpClient httpClient = new(httpHandler); +#pragma warning restore CA5399 + + AIProjectClientOptions clientOptions = new() + { + Transport = new HttpClientPipelineTransport(httpClient) + }; + + FoundryAgent agent = new( + s_testEndpoint, + new FakeAuthenticationTokenProvider(), + model: "gpt-4o-mini", + instructions: "Test", + clientOptions: clientOptions); + + AgentSession session = await agent.CreateSessionAsync(); + await agent.RunAsync("Hello", session); + + Assert.True(userAgentFound, "Expected MEAI user-agent header to be present in requests."); + } + + #endregion +} diff --git a/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/TestDataUtil.cs b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/TestDataUtil.cs index 0a33c03ccd..9cd2ecea46 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/TestDataUtil.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AzureAI.UnitTests/TestDataUtil.cs @@ -63,8 +63,8 @@ internal static class TestDataUtil json = ApplyInstructions(json, instructions); json = ApplyDescription(json, description); // Remove the version and id fields to simulate hosted agents without version - json = json.Replace("\"version\": \"1\",", "\"version\": \"\","); - json = json.Replace("\"id\": \"agent_abc123:1\",", "\"id\": \"\","); + json = json.Replace("\"version\": \"1\",", "\"version\": \"\",") + .Replace("\"id\": \"agent_abc123:1\",", "\"id\": \"\","); return json; } @@ -79,8 +79,8 @@ internal static class TestDataUtil json = ApplyInstructions(json, instructions); json = ApplyDescription(json, description); // Remove the version and id fields to simulate hosted agents without version - json = json.Replace("\"version\": \"1\",", "\"version\": \"\","); - json = json.Replace("\"id\": \"agent_abc123:1\",", "\"id\": \"\","); + json = json.Replace("\"version\": \"1\",", "\"version\": \"\",") + .Replace("\"id\": \"agent_abc123:1\",", "\"id\": \"\","); return json; } diff --git a/dotnet/tests/Microsoft.Agents.AI.FoundryMemory.IntegrationTests/FoundryMemoryProviderTests.cs b/dotnet/tests/Microsoft.Agents.AI.FoundryMemory.IntegrationTests/FoundryMemoryProviderTests.cs index 9b3c95c5c2..d6092f5231 100644 --- a/dotnet/tests/Microsoft.Agents.AI.FoundryMemory.IntegrationTests/FoundryMemoryProviderTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.FoundryMemory.IntegrationTests/FoundryMemoryProviderTests.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +#pragma warning disable CS0618 // Tests intentionally exercise obsolete extension methods + using System; using System.Threading.Tasks; using Azure.AI.Projects; diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInMemorySkillsSourceTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInMemorySkillsSourceTests.cs new file mode 100644 index 0000000000..69e96f0957 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInMemorySkillsSourceTests.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Agents.AI.UnitTests.AgentSkills; + +/// +/// Unit tests for . +/// +public sealed class AgentInMemorySkillsSourceTests +{ + [Fact] + public async Task GetSkillsAsync_ValidSkills_ReturnsAllAsync() + { + // Arrange + var skills = new AgentSkill[] + { + new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."), + new AgentInlineSkill("another", "Another valid skill.", "More instructions."), + }; + var source = new AgentInMemorySkillsSource(skills); + + // Act + var result = await source.GetSkillsAsync(CancellationToken.None); + + // Assert + Assert.Equal(2, result.Count); + Assert.Equal("my-skill", result[0].Frontmatter.Name); + Assert.Equal("another", result[1].Frontmatter.Name); + } + + [Theory] + [InlineData("INVALID-NAME")] + [InlineData("-leading")] + [InlineData("trailing-")] + public void Constructor_InvalidFrontmatter_ThrowsArgumentException(string invalidName) + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkill(invalidName, "A skill.", "Instructions.")); + } + + [Fact] + public void Constructor_NullSkills_Throws() + { + // Act & Assert + Assert.Throws(() => new AgentInMemorySkillsSource(null!)); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillResourceTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillResourceTests.cs new file mode 100644 index 0000000000..202a90d460 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillResourceTests.cs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Agents.AI.UnitTests.AgentSkills; + +/// +/// Unit tests for . +/// +public sealed class AgentInlineSkillResourceTests +{ + [Fact] + public async Task ReadAsync_StaticValue_ReturnsValueAsync() + { + // Arrange + var resource = new AgentInlineSkillResource("config", "my-value"); + + // Act + var result = await resource.ReadAsync(); + + // Assert + Assert.Equal("my-value", result); + } + + [Fact] + public async Task ReadAsync_StaticObjectValue_ReturnsSameInstanceAsync() + { + // Arrange + var obj = new object(); + var resource = new AgentInlineSkillResource("ref", obj); + + // Act + var result = await resource.ReadAsync(); + + // Assert + Assert.Same(obj, result); + } + + [Fact] + public async Task ReadAsync_Delegate_InvokesFunctionAsync() + { + // Arrange + int callCount = 0; + var resource = new AgentInlineSkillResource("dynamic", () => + { + callCount++; + return "computed"; + }); + + // Act + var result = await resource.ReadAsync(); + + // Assert + Assert.Equal("computed", result?.ToString()); + Assert.Equal(1, callCount); + } + + [Fact] + public async Task ReadAsync_Delegate_InvokesEachTimeAsync() + { + // Arrange + int callCount = 0; + var resource = new AgentInlineSkillResource("counter", () => ++callCount); + + // Act + await resource.ReadAsync(); + await resource.ReadAsync(); + var result = await resource.ReadAsync(); + + // Assert + Assert.Equal(3, callCount); + } + + [Fact] + public void Constructor_StaticValue_SetsNameAndDescription() + { + // Arrange & Act + var resource = new AgentInlineSkillResource("my-res", "val", "A description."); + + // Assert + Assert.Equal("my-res", resource.Name); + Assert.Equal("A description.", resource.Description); + } + + [Fact] + public void Constructor_StaticValue_NullDescription_DescriptionIsNull() + { + // Arrange & Act + var resource = new AgentInlineSkillResource("my-res", "val"); + + // Assert + Assert.Null(resource.Description); + } + + [Fact] + public void Constructor_StaticValue_NullValue_Throws() + { + // Act & Assert — cast needed to target the object overload +#pragma warning disable IDE0004 + Assert.Throws(() => + new AgentInlineSkillResource("my-res", (object)null!)); +#pragma warning restore IDE0004 + } + + [Fact] + public void Constructor_Delegate_NullMethod_Throws() + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkillResource("my-res", null!)); + } + + [Fact] + public void Constructor_NullName_Throws() + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkillResource(null!, "val")); + } + + [Fact] + public void Constructor_WhitespaceName_Throws() + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkillResource(" ", "val")); + } + + [Fact] + public void Constructor_Delegate_SetsNameAndDescription() + { + // Arrange & Act + var resource = new AgentInlineSkillResource("dyn-res", () => "hello", "Dynamic resource."); + + // Assert + Assert.Equal("dyn-res", resource.Name); + Assert.Equal("Dynamic resource.", resource.Description); + } + + [Fact] + public async Task ReadAsync_SupportsCancellationTokenAsync() + { + // Arrange + using var cts = new CancellationTokenSource(); + var resource = new AgentInlineSkillResource("cancellable", "value"); + + // Act — should not throw with a non-cancelled token + var result = await resource.ReadAsync(cancellationToken: cts.Token); + + // Assert + Assert.Equal("value", result); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillScriptTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillScriptTests.cs new file mode 100644 index 0000000000..61f42ca66d --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillScriptTests.cs @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.UnitTests.AgentSkills; + +/// +/// Unit tests for . +/// +public sealed class AgentInlineSkillScriptTests +{ + [Fact] + public async Task RunAsync_InvokesDelegate_ReturnsResultAsync() + { + // Arrange + var script = new AgentInlineSkillScript("greet", () => "hello"); + var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions."); + + // Act + var result = await script.RunAsync(skill, new AIFunctionArguments(), CancellationToken.None); + + // Assert + Assert.Equal("hello", result?.ToString()); + } + + [Fact] + public async Task RunAsync_WithParameters_PassesArgumentsAsync() + { + // Arrange + var script = new AgentInlineSkillScript("add", (int a, int b) => a + b); + var skill = new AgentInlineSkill("calc-skill", "Calc.", "Instructions."); + var args = new AIFunctionArguments { ["a"] = 3, ["b"] = 7 }; + + // Act + var result = await script.RunAsync(skill, args, CancellationToken.None); + + // Assert + Assert.Equal(10, int.Parse(result?.ToString()!)); + } + + [Fact] + public void ParametersSchema_NoParameters_ReturnsSchema() + { + // Arrange + var script = new AgentInlineSkillScript("noop", () => "ok"); + + // Act + var schema = script.ParametersSchema; + + // Assert — parameterless delegates still produce a schema + Assert.NotNull(schema); + } + + [Fact] + public void ParametersSchema_WithParameters_ContainsPropertyNames() + { + // Arrange + var script = new AgentInlineSkillScript("search", (string query, int limit) => $"{query}:{limit}"); + + // Act + var schema = script.ParametersSchema; + + // Assert + Assert.NotNull(schema); + var schemaText = schema!.Value.GetRawText(); + Assert.Contains("query", schemaText); + Assert.Contains("limit", schemaText); + } + + [Fact] + public void Constructor_SetsNameAndDescription() + { + // Arrange & Act + var script = new AgentInlineSkillScript("my-script", () => "ok", "Does something."); + + // Assert + Assert.Equal("my-script", script.Name); + Assert.Equal("Does something.", script.Description); + } + + [Fact] + public void Constructor_NullDescription_DescriptionIsNull() + { + // Arrange & Act + var script = new AgentInlineSkillScript("my-script", () => "ok"); + + // Assert + Assert.Null(script.Description); + } + + [Fact] + public void Constructor_NullName_Throws() + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkillScript(null!, () => "ok")); + } + + [Fact] + public void Constructor_WhitespaceName_Throws() + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkillScript(" ", () => "ok")); + } + + [Fact] + public void Constructor_NullMethod_Throws() + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkillScript("my-script", null!)); + } + + [Fact] + public async Task RunAsync_StringParameter_WorksAsync() + { + // Arrange + var script = new AgentInlineSkillScript("echo", (string message) => message); + var skill = new AgentInlineSkill("test-skill", "Test.", "Instructions."); + var args = new AIFunctionArguments { ["message"] = "hello world" }; + + // Act + var result = await script.RunAsync(skill, args, CancellationToken.None); + + // Assert + Assert.Equal("hello world", result?.ToString()); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillTests.cs new file mode 100644 index 0000000000..9a1f6a4951 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentInlineSkillTests.cs @@ -0,0 +1,420 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.UnitTests.AgentSkills; + +/// +/// Unit tests for . +/// +public sealed class AgentInlineSkillTests +{ + [Fact] + public void Constructor_WithNameAndDescription_SetsFrontmatter() + { + // Arrange & Act + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Assert + Assert.Equal("my-skill", skill.Frontmatter.Name); + Assert.Equal("A valid skill.", skill.Frontmatter.Description); + Assert.Null(skill.Frontmatter.License); + Assert.Null(skill.Frontmatter.Compatibility); + Assert.Null(skill.Frontmatter.AllowedTools); + Assert.Null(skill.Frontmatter.Metadata); + } + + [Fact] + public void Constructor_WithAllProps_SetsFrontmatter() + { + // Arrange + var metadata = new AdditionalPropertiesDictionary { ["key"] = "value" }; + + // Act + var skill = new AgentInlineSkill( + "my-skill", + "A valid skill.", + "Instructions.", + license: "MIT", + compatibility: "gpt-4", + allowedTools: "tool-a tool-b", + metadata: metadata); + + // Assert + Assert.Equal("my-skill", skill.Frontmatter.Name); + Assert.Equal("A valid skill.", skill.Frontmatter.Description); + Assert.Equal("MIT", skill.Frontmatter.License); + Assert.Equal("gpt-4", skill.Frontmatter.Compatibility); + Assert.Equal("tool-a tool-b", skill.Frontmatter.AllowedTools); + Assert.NotNull(skill.Frontmatter.Metadata); + Assert.Equal("value", skill.Frontmatter.Metadata["key"]); + } + + [Fact] + public void Constructor_WithFrontmatter_UsesFrontmatterDirectly() + { + // Arrange + var frontmatter = new AgentSkillFrontmatter("my-skill", "A valid skill.") + { + License = "Apache-2.0", + Compatibility = "gpt-4", + AllowedTools = "tool-a", + Metadata = new AdditionalPropertiesDictionary { ["env"] = "prod" }, + }; + + // Act + var skill = new AgentInlineSkill(frontmatter, "Instructions."); + + // Assert + Assert.Same(frontmatter, skill.Frontmatter); + Assert.Equal("Apache-2.0", skill.Frontmatter.License); + Assert.Equal("gpt-4", skill.Frontmatter.Compatibility); + Assert.Equal("tool-a", skill.Frontmatter.AllowedTools); + Assert.Equal("prod", skill.Frontmatter.Metadata!["env"]); + } + + [Fact] + public void Constructor_WithFrontmatter_NullFrontmatter_Throws() + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkill(null!, "Instructions.")); + } + + [Fact] + public void Constructor_WithFrontmatter_NullInstructions_Throws() + { + // Arrange + var frontmatter = new AgentSkillFrontmatter("my-skill", "A valid skill."); + + // Act & Assert + Assert.Throws(() => + new AgentInlineSkill(frontmatter, null!)); + } + + [Fact] + public void Constructor_WithAllProps_NullInstructions_Throws() + { + // Act & Assert + Assert.Throws(() => + new AgentInlineSkill("my-skill", "A valid skill.", null!)); + } + + [Fact] + public void Content_ContainsNameDescriptionAndInstructions() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Do the thing."); + + // Act + var content = skill.Content; + + // Assert + Assert.Contains("my-skill", content); + Assert.Contains("A valid skill.", content); + Assert.Contains("\nDo the thing.\n", content); + } + + [Fact] + public void Content_EscapesXmlCharacters() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "xz\"w & it's more", "1 & 2 < 3"); + + // Act + var content = skill.Content; + + // Assert + Assert.Contains("my-skill", content); + Assert.Contains("x<y>z"w & it's more", content); + Assert.Contains("1 & 2 < 3", content); // instructions are escaped + } + + [Fact] + public void Content_IsCachedAcrossAccesses() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act + var first = skill.Content; + var second = skill.Content; + + // Assert + Assert.Same(first, second); + } + + [Fact] + public void Content_IncludesResourcesAddedBeforeFirstAccess() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddResource("config", "value1", "A config resource."); + + // Act + var content = skill.Content; + + // Assert + Assert.Contains("", content); + Assert.Contains("config", content); + } + + [Fact] + public void Content_IncludesDelegateResourcesAddedBeforeFirstAccess() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddResource("dynamic", () => "hello"); + + // Act + var content = skill.Content; + + // Assert + Assert.Contains("", content); + Assert.Contains("dynamic", content); + } + + [Fact] + public void Content_IncludesScriptsAddedBeforeFirstAccess() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddScript("run", () => "result", "Runs something."); + + // Act + var content = skill.Content; + + // Assert + Assert.Contains("", content); + Assert.Contains("run", content); + } + + [Fact] + public void Content_IsCachedAndNotRebuilt() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddResource("r1", "v1"); + + // Act + var first = skill.Content; + var second = skill.Content; + + // Assert + Assert.Same(first, second); + } + + [Fact] + public void Content_IncludesResourcesAndScriptsAddedBeforeFirstAccess() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddResource("r1", "v1"); + skill.AddScript("s1", () => "ok"); + + // Act + var content = skill.Content; + + // Assert + Assert.Contains("", content); + Assert.Contains("r1", content); + Assert.Contains("", content); + Assert.Contains("s1", content); + } + + [Fact] + public void Content_ParametersSchema_IsXmlEscaped() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddScript("search", (string query, int limit) => $"found {limit} results for {query}"); + + // Act + var content = skill.Content; + + // Assert — JSON schema should be present and XML content chars escaped + Assert.Contains("parameters_schema", content); + Assert.DoesNotContain("(() => skill.AddResource("config", (object)null!)); +#pragma warning restore IDE0004 + } + + [Fact] + public void AddResource_NullDelegate_Throws() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act & Assert + Assert.Throws(() => skill.AddResource("config", null!)); + } + + [Fact] + public void AddScript_NullDelegate_Throws() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act & Assert + Assert.Throws(() => skill.AddScript("run", null!)); + } + + [Fact] + public void Resources_WhenNoneAdded_ReturnsNull() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act & Assert + Assert.Null(skill.Resources); + } + + [Fact] + public void Scripts_WhenNoneAdded_ReturnsNull() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act & Assert + Assert.Null(skill.Scripts); + } + + [Fact] + public void AddResource_ReturnsSameInstance_ForChaining() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act + var returned = skill.AddResource("r1", "v1"); + + // Assert + Assert.Same(skill, returned); + } + + [Fact] + public void AddResource_Delegate_ReturnsSameInstance_ForChaining() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act + var returned = skill.AddResource("r1", () => "v1"); + + // Assert + Assert.Same(skill, returned); + } + + [Fact] + public void AddScript_ReturnsSameInstance_ForChaining() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act + var returned = skill.AddScript("s1", () => "ok"); + + // Assert + Assert.Same(skill, returned); + } + + [Fact] + public void Content_NoResourcesOrScripts_DoesNotContainResourcesOrScriptsTags() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + + // Act + var content = skill.Content; + + // Assert + Assert.DoesNotContain("", content); + Assert.DoesNotContain("", content); + } + + [Fact] + public void Content_ResourcesAddedAfterCaching_AreNotIncluded() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + _ = skill.Content; // trigger caching + skill.AddResource("late-resource", "late-value"); + + // Act + var content = skill.Content; + + // Assert — the late resource should not appear because content was cached + Assert.DoesNotContain("late-resource", content); + } + + [Fact] + public void Content_ScriptsAddedAfterCaching_AreNotIncluded() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + _ = skill.Content; // trigger caching + skill.AddScript("late-script", () => "late"); + + // Act + var content = skill.Content; + + // Assert — the late script should not appear because content was cached + Assert.DoesNotContain("late-script", content); + } + + [Fact] + public void Content_ScriptWithDescription_IncludesDescriptionAttribute() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddScript("my-script", () => "ok", "Runs something."); + + // Act + var content = skill.Content; + + // Assert + Assert.Contains("description=\"Runs something.\"", content); + } + + [Fact] + public void Content_ScriptWithoutParametersOrDescription_UsesSelfClosingTag() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddScript("simple", () => "ok"); + + // Act + var content = skill.Content; + + // Assert — parameterless Action delegates still produce a schema, so this + // verifies the script is at least included in the output + Assert.Contains("simple", content); + } + + [Fact] + public void Content_ResourceWithDescription_IncludesDescriptionAttribute() + { + // Arrange + var skill = new AgentInlineSkill("my-skill", "A valid skill.", "Instructions."); + skill.AddResource("with-desc", "value", "A described resource."); + skill.AddResource("no-desc", "value"); + + // Act + var content = skill.Content; + + // Assert + Assert.Contains("description=\"A described resource.\"", content); + Assert.DoesNotContain("no-desc\" description", content); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentSkillsProviderTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentSkillsProviderTests.cs index 6dfc45918a..87e98b3da3 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentSkillsProviderTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/AgentSkillsProviderTests.cs @@ -270,7 +270,7 @@ public sealed class AgentSkillsProviderTests : IDisposable // Arrange var source = new CountingAgentSkillsSource( [ - new TestAgentSkill("concurrent-skill", "Concurrent test", "Body.") + new AgentInlineSkill("concurrent-skill", "Concurrent test", "Body.") ]); var provider = new AgentSkillsProvider(source); @@ -502,7 +502,7 @@ public sealed class AgentSkillsProviderTests : IDisposable // Arrange var source = new CountingAgentSkillsSource( [ - new TestAgentSkill("no-cache-skill", "No cache test", "Body.") + new AgentInlineSkill("no-cache-skill", "No cache test", "Body.") ]); var provider = new AgentSkillsProviderBuilder() .UseSource(source) @@ -525,7 +525,7 @@ public sealed class AgentSkillsProviderTests : IDisposable // Arrange var source = new CountingAgentSkillsSource( [ - new TestAgentSkill("cached-skill", "Cached test", "Body.") + new AgentInlineSkill("cached-skill", "Cached test", "Body.") ]); var provider = new AgentSkillsProviderBuilder() .UseSource(source) @@ -547,7 +547,7 @@ public sealed class AgentSkillsProviderTests : IDisposable // Arrange var source = new CountingAgentSkillsSource( [ - new TestAgentSkill("default-skill", "Default test", "Body.") + new AgentInlineSkill("default-skill", "Default test", "Body.") ]); var provider = new AgentSkillsProviderBuilder() .UseSource(source) @@ -563,6 +563,78 @@ public sealed class AgentSkillsProviderTests : IDisposable Assert.Equal(1, source.GetSkillsCallCount); } + [Fact] + public async Task Build_PreservesSourceRegistrationOrderAsync() + { + // Arrange — register file, inline, file in that order + string dir1 = Path.Combine(this._testRoot, "dir1"); + string dir2 = Path.Combine(this._testRoot, "dir2"); + CreateSkillIn(dir1, "file-skill-1", "First file skill", "Body 1."); + CreateSkillIn(dir2, "file-skill-2", "Second file skill", "Body 2."); + + var inlineSkill = new AgentInlineSkill("inline-skill", "Inline skill", "Body inline."); + + var provider = new AgentSkillsProviderBuilder() + .UseFileSkill(dir1) + .UseSkills(inlineSkill) + .UseFileSkill(dir2) + .UseFileScriptRunner(s_noOpExecutor) + .UseOptions(o => o.DisableCaching = true) + .Build(); + + var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext()); + + // Act + var result = await provider.InvokingAsync(invokingContext, CancellationToken.None); + + // Assert — all three skills should be present in alphabetical order in the prompt + Assert.NotNull(result.Instructions); + var instructions = result.Instructions!; + var indexFileSkill1 = instructions.IndexOf("file-skill-1", StringComparison.Ordinal); + var indexFileSkill2 = instructions.IndexOf("file-skill-2", StringComparison.Ordinal); + var indexInlineSkill = instructions.IndexOf("inline-skill", StringComparison.Ordinal); + + Assert.True(indexFileSkill1 >= 0, "file-skill-1 should be present in the instructions."); + Assert.True(indexFileSkill2 >= 0, "file-skill-2 should be present in the instructions."); + Assert.True(indexInlineSkill >= 0, "inline-skill should be present in the instructions."); + + Assert.True(indexFileSkill1 < indexFileSkill2, "file-skill-1 should appear before file-skill-2."); + Assert.True(indexFileSkill2 < indexInlineSkill, "file-skill-2 should appear before inline-skill."); + } + + [Fact] + public async Task Build_MixedSources_AllSkillsDiscoveredAsync() + { + // Arrange — use UseSource, UseSkill, and UseFileSkill in mixed order + string dir = Path.Combine(this._testRoot, "mixed-dir"); + CreateSkillIn(dir, "file-skill", "File skill", "Body file."); + + var inlineSkill = new AgentInlineSkill("inline-skill", "Inline skill", "Body inline."); + var customSource = new CountingAgentSkillsSource( + [ + new AgentInlineSkill("custom-skill", "Custom source skill", "Body custom.") + ]); + + var provider = new AgentSkillsProviderBuilder() + .UseSource(customSource) + .UseSkills(inlineSkill) + .UseFileSkill(dir) + .UseFileScriptRunner(s_noOpExecutor) + .UseOptions(o => o.DisableCaching = true) + .Build(); + + var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext()); + + // Act + var result = await provider.InvokingAsync(invokingContext, CancellationToken.None); + + // Assert — all skills from all sources are present + Assert.NotNull(result.Instructions); + Assert.Contains("custom-skill", result.Instructions); + Assert.Contains("inline-skill", result.Instructions); + Assert.Contains("file-skill", result.Instructions); + } + [Fact] public async Task InvokingCoreAsync_WithScriptsAndScriptApproval_WrapsRunScriptToolAsync() { @@ -722,6 +794,63 @@ public sealed class AgentSkillsProviderTests : IDisposable Assert.Contains("Body 1.", content!.ToString()!); } + [Fact] + public async Task Constructor_InlineSkillsParams_ProvidesSkillsAsync() + { + // Arrange + var skill1 = new AgentInlineSkill("inline-a", "Inline A", "Instructions A."); + var skill2 = new AgentInlineSkill("inline-b", "Inline B", "Instructions B."); + var provider = new AgentSkillsProvider(skill1, skill2); + var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext()); + + // Act + var result = await provider.InvokingAsync(invokingContext, CancellationToken.None); + + // Assert + Assert.NotNull(result.Instructions); + Assert.Contains("inline-a", result.Instructions); + Assert.Contains("inline-b", result.Instructions); + } + + [Fact] + public async Task Constructor_InlineSkillsEnumerable_ProvidesSkillsAsync() + { + // Arrange + var skills = new List + { + new("enum-inline-a", "Inline A", "Instructions A."), + new("enum-inline-b", "Inline B", "Instructions B."), + }; + var provider = new AgentSkillsProvider(skills); + var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext()); + + // Act + var result = await provider.InvokingAsync(invokingContext, CancellationToken.None); + + // Assert + Assert.NotNull(result.Instructions); + Assert.Contains("enum-inline-a", result.Instructions); + Assert.Contains("enum-inline-b", result.Instructions); + } + + [Fact] + public async Task Constructor_InlineSkills_DeduplicatesAsync() + { + // Arrange — two inline skills with the same name + var skill1 = new AgentInlineSkill("dup-inline", "First", "First instructions."); + var skill2 = new AgentInlineSkill("dup-inline", "Second", "Second instructions."); + var provider = new AgentSkillsProvider(skill1, skill2); + var invokingContext = new AIContextProvider.InvokingContext(this._agent, session: null, new AIContext()); + + // Act + var result = await provider.InvokingAsync(invokingContext, CancellationToken.None); + var loadSkillTool = result.Tools!.First(t => t.Name == "load_skill") as AIFunction; + var content = await loadSkillTool!.InvokeAsync(new AIFunctionArguments(new Dictionary { ["skillName"] = "dup-inline" })); + + // Assert — only one occurrence (first) + Assert.Contains("First instructions.", content!.ToString()!); + } + /// /// A test skill source that counts how many times is called. /// @@ -743,23 +872,4 @@ public sealed class AgentSkillsProviderTests : IDisposable return Task.FromResult(this._skills); } } - - private sealed class TestAgentSkill : AgentSkill - { - private readonly string _content; - - public TestAgentSkill(string name, string description, string content) - { - this.Frontmatter = new AgentSkillFrontmatter(name, description); - this._content = content; - } - - public override AgentSkillFrontmatter Frontmatter { get; } - - public override string Content => this._content; - - public override IReadOnlyList? Resources => null; - - public override IReadOnlyList? Scripts => null; - } } diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/DeduplicatingAgentSkillsSourceTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/DeduplicatingAgentSkillsSourceTests.cs index 860f402005..7895023681 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/DeduplicatingAgentSkillsSourceTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/DeduplicatingAgentSkillsSourceTests.cs @@ -16,9 +16,11 @@ public sealed class DeduplicatingAgentSkillsSourceTests public async Task GetSkillsAsync_NoDuplicates_ReturnsAllSkillsAsync() { // Arrange - var inner = new TestAgentSkillsSource( - new TestAgentSkill("skill-a", "A", "Instructions A."), - new TestAgentSkill("skill-b", "B", "Instructions B.")); + var inner = new AgentInMemorySkillsSource(new AgentSkill[] + { + new AgentInlineSkill("skill-a", "A", "Instructions A."), + new AgentInlineSkill("skill-b", "B", "Instructions B."), + }); var source = new DeduplicatingAgentSkillsSource(inner); // Act @@ -34,11 +36,11 @@ public sealed class DeduplicatingAgentSkillsSourceTests // Arrange var skills = new AgentSkill[] { - new TestAgentSkill("dupe", "First", "Instructions 1."), - new TestAgentSkill("dupe", "Second", "Instructions 2."), - new TestAgentSkill("unique", "Unique", "Instructions 3."), + new AgentInlineSkill("dupe", "First", "Instructions 1."), + new AgentInlineSkill("dupe", "Second", "Instructions 2."), + new AgentInlineSkill("unique", "Unique", "Instructions 3."), }; - var inner = new TestAgentSkillsSource(skills); + var inner = new AgentInMemorySkillsSource(skills); var source = new DeduplicatingAgentSkillsSource(inner); // Act @@ -53,7 +55,7 @@ public sealed class DeduplicatingAgentSkillsSourceTests [Fact] public async Task GetSkillsAsync_CaseInsensitiveDuplication_KeepsFirstAsync() { - // Arrange — use a custom source that returns skills with same name but different casing + // Arrange - Use a custom source that returns skills with same name but different casing var inner = new FakeDuplicateCaseSource(); var source = new DeduplicatingAgentSkillsSource(inner); @@ -69,7 +71,7 @@ public sealed class DeduplicatingAgentSkillsSourceTests public async Task GetSkillsAsync_EmptySource_ReturnsEmptyAsync() { // Arrange - var inner = new TestAgentSkillsSource(System.Array.Empty()); + var inner = new AgentInMemorySkillsSource(System.Array.Empty()); var source = new DeduplicatingAgentSkillsSource(inner); // Act @@ -90,8 +92,8 @@ public sealed class DeduplicatingAgentSkillsSourceTests // two skills with the same lowercase name to test case-insensitive dedup. var skills = new List { - new TestAgentSkill("my-skill", "First", "Instructions 1."), - new TestAgentSkill("my-skill", "Second", "Instructions 2."), + new AgentInlineSkill("my-skill", "First", "Instructions 1."), + new AgentInlineSkill("my-skill", "Second", "Instructions 2."), }; return Task.FromResult>(skills); } diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/FilteringAgentSkillsSourceTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/FilteringAgentSkillsSourceTests.cs index de145004e0..12bdb28e05 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/FilteringAgentSkillsSourceTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/AgentSkills/FilteringAgentSkillsSourceTests.cs @@ -15,9 +15,11 @@ public sealed class FilteringAgentSkillsSourceTests public async Task GetSkillsAsync_PredicateIncludesAll_ReturnsAllSkillsAsync() { // Arrange - var inner = new TestAgentSkillsSource( - new TestAgentSkill("skill-a", "A", "Instructions A."), - new TestAgentSkill("skill-b", "B", "Instructions B.")); + var inner = new AgentInMemorySkillsSource(new AgentSkill[] + { + new AgentInlineSkill("skill-a", "A", "Instructions A."), + new AgentInlineSkill("skill-b", "B", "Instructions B."), + }); var source = new FilteringAgentSkillsSource(inner, _ => true); // Act @@ -31,9 +33,11 @@ public sealed class FilteringAgentSkillsSourceTests public async Task GetSkillsAsync_PredicateExcludesAll_ReturnsEmptyAsync() { // Arrange - var inner = new TestAgentSkillsSource( - new TestAgentSkill("skill-a", "A", "Instructions A."), - new TestAgentSkill("skill-b", "B", "Instructions B.")); + var inner = new AgentInMemorySkillsSource(new AgentSkill[] + { + new AgentInlineSkill("skill-a", "A", "Instructions A."), + new AgentInlineSkill("skill-b", "B", "Instructions B."), + }); var source = new FilteringAgentSkillsSource(inner, _ => false); // Act @@ -47,10 +51,12 @@ public sealed class FilteringAgentSkillsSourceTests public async Task GetSkillsAsync_PartialFilter_ReturnsMatchingSkillsOnlyAsync() { // Arrange - var inner = new TestAgentSkillsSource( - new TestAgentSkill("keep-me", "Keep", "Instructions."), - new TestAgentSkill("drop-me", "Drop", "Instructions."), - new TestAgentSkill("keep-also", "KeepAlso", "Instructions.")); + var inner = new AgentInMemorySkillsSource(new AgentSkill[] + { + new AgentInlineSkill("keep-me", "Keep", "Instructions."), + new AgentInlineSkill("drop-me", "Drop", "Instructions."), + new AgentInlineSkill("keep-also", "KeepAlso", "Instructions."), + }); var source = new FilteringAgentSkillsSource( inner, skill => skill.Frontmatter.Name.StartsWith("keep", StringComparison.OrdinalIgnoreCase)); @@ -67,7 +73,7 @@ public sealed class FilteringAgentSkillsSourceTests public async Task GetSkillsAsync_EmptySource_ReturnsEmptyAsync() { // Arrange - var inner = new TestAgentSkillsSource(Array.Empty()); + var inner = new AgentInMemorySkillsSource(Array.Empty()); var source = new FilteringAgentSkillsSource(inner, _ => true); // Act @@ -81,7 +87,7 @@ public sealed class FilteringAgentSkillsSourceTests public void Constructor_NullPredicate_Throws() { // Arrange - var inner = new TestAgentSkillsSource(Array.Empty()); + var inner = new AgentInMemorySkillsSource(Array.Empty()); // Act & Assert Assert.Throws(() => new FilteringAgentSkillsSource(inner, null!)); @@ -98,11 +104,13 @@ public sealed class FilteringAgentSkillsSourceTests public async Task GetSkillsAsync_PreservesOrderAsync() { // Arrange - var inner = new TestAgentSkillsSource( - new TestAgentSkill("alpha", "Alpha", "Instructions."), - new TestAgentSkill("beta", "Beta", "Instructions."), - new TestAgentSkill("gamma", "Gamma", "Instructions."), - new TestAgentSkill("delta", "Delta", "Instructions.")); + var inner = new AgentInMemorySkillsSource(new AgentSkill[] + { + new AgentInlineSkill("alpha", "Alpha", "Instructions."), + new AgentInlineSkill("beta", "Beta", "Instructions."), + new AgentInlineSkill("gamma", "Gamma", "Instructions."), + new AgentInlineSkill("delta", "Delta", "Instructions."), + }); // Keep only alpha and gamma var source = new FilteringAgentSkillsSource( diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgentTestHelper.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgentTestHelper.cs index a3d2bd0c6a..b3296a1306 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgentTestHelper.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgentTestHelper.cs @@ -14,7 +14,7 @@ namespace Microsoft.Agents.AI.UnitTests; /// /// Shared test helper for integration tests that verify -/// end-to-end behavior with and +/// end-to-end behavior with and /// . /// internal static class ChatClientAgentTestHelper diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgentTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgentTests.cs index 7241ca763e..3db8e0c046 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgentTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgentTests.cs @@ -379,12 +379,10 @@ public partial class ChatClientAgentTests } /// - /// Verify that RunAsync passes ChatOptions with null ConversationId when using regular AgentRunOptions. - /// When per-service-call persistence is active (default), the sentinel conversation ID is set on ChatOptions - /// and then stripped by ChatHistoryPersistingChatClient before reaching the inner client. + /// Verify that RunAsync passes null ChatOptions when using regular AgentRunOptions. /// [Fact] - public async Task RunAsyncPassesChatOptionsWithNullConversationIdWhenUsingRegularAgentRunOptionsAsync() + public async Task RunAsyncPassesNullChatOptionsWhenUsingRegularAgentRunOptionsAsync() { // Arrange ChatOptions? capturedOptions = null; @@ -403,9 +401,8 @@ public partial class ChatClientAgentTests // Act await agent.RunAsync([new(ChatRole.User, "test")], options: runOptions); - // Assert — the inner client receives ChatOptions with null ConversationId (sentinel was stripped) - Assert.NotNull(capturedOptions); - Assert.Null(capturedOptions!.ConversationId); + // Assert + Assert.Null(capturedOptions); } /// @@ -1793,6 +1790,75 @@ public partial class ChatClientAgentTests Times.Once); } + /// + /// Verify that RunStreamingAsync passes through null MessageId from provider without modification. + /// MessageId generation is handled by downstream consumers (e.g., AGUI layer), not ChatClientAgent. + /// + [Fact] + public async Task RunStreamingAsync_WithNullMessageId_PassesThroughNullAsync() + { + // Arrange - Provider returns updates WITHOUT MessageId + ChatResponseUpdate[] returnUpdates = + [ + new ChatResponseUpdate(role: ChatRole.Assistant, content: "Hello"), + new ChatResponseUpdate(role: ChatRole.Assistant, content: " world"), + ]; + + Mock mockService = new(); + mockService.Setup( + s => s.GetStreamingResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())).Returns(ToAsyncEnumerableAsync(returnUpdates)); + + ChatClientAgent agent = new(mockService.Object); + + // Act + List result = []; + await foreach (var update in agent.RunStreamingAsync([new ChatMessage(ChatRole.User, "Hi")])) + { + result.Add(update); + } + + // Assert - MessageId should be null (ChatClientAgent does not generate fallback IDs) + Assert.Equal(2, result.Count); + Assert.All(result, u => Assert.Null(u.MessageId)); + } + + /// + /// Verify that RunStreamingAsync preserves provider-supplied MessageId when present. + /// + [Fact] + public async Task RunStreamingAsync_WithProviderMessageId_PreservesItAsync() + { + // Arrange - Provider returns updates WITH MessageId (like OpenAI) + ChatResponseUpdate[] returnUpdates = + [ + new ChatResponseUpdate(role: ChatRole.Assistant, content: "Hello") { MessageId = "chatcmpl-abc123" }, + new ChatResponseUpdate(role: ChatRole.Assistant, content: " world") { MessageId = "chatcmpl-abc123" }, + ]; + + Mock mockService = new(); + mockService.Setup( + s => s.GetStreamingResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())).Returns(ToAsyncEnumerableAsync(returnUpdates)); + + ChatClientAgent agent = new(mockService.Object); + + // Act + List result = []; + await foreach (var update in agent.RunStreamingAsync([new ChatMessage(ChatRole.User, "Hi")])) + { + result.Add(update); + } + + // Assert - Provider's MessageId should be preserved, not overwritten + Assert.Equal(2, result.Count); + Assert.All(result, u => Assert.Equal("chatcmpl-abc123", u.MessageId)); + } + /// /// Verify that RunStreamingAsync uses the ChatHistoryProvider factory when the chat client returns no conversation id. /// diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ApprovalsTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ApprovalsTests.cs index 6300942c9d..f3550359dc 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ApprovalsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ApprovalsTests.cs @@ -9,7 +9,7 @@ namespace Microsoft.Agents.AI.UnitTests; /// /// Contains unit tests that verify the end-to-end approval flow behavior of the -/// class with , +/// class with , /// ensuring that chat history is correctly persisted across multi-turn approval interactions. /// public class ChatClientAgent_ApprovalsTests @@ -48,7 +48,7 @@ public class ChatClientAgent_ApprovalsTests agentOptions: new() { ChatOptions = new() { Tools = [approvalTool] }, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }, callIndex: callIndex, capturedInputs: capturedInputs); @@ -123,7 +123,6 @@ public class ChatClientAgent_ApprovalsTests agentOptions: new() { ChatOptions = new() { Tools = [approvalTool] }, - PersistChatHistoryAtEndOfRun = true, }, callIndex: callIndex, capturedInputs: capturedInputs); @@ -150,8 +149,10 @@ public class ChatClientAgent_ApprovalsTests expectedHistory: [ // End-of-run persistence retains the approval request from Turn 1 + // and the approval response from Turn 2 new(ChatRole.User, TextContains: "What's the weather?"), new(ChatRole.Assistant, ContentTypes: [typeof(ToolApprovalRequestContent)]), + new(ChatRole.User, ContentTypes: [typeof(ToolApprovalResponseContent)]), new(ChatRole.Assistant, ContentTypes: [typeof(FunctionCallContent)]), new(ChatRole.Tool, ContentTypes: [typeof(FunctionResultContent)]), new(ChatRole.Assistant, TextContains: "sunny and 22°C"), @@ -196,7 +197,6 @@ public class ChatClientAgent_ApprovalsTests agentOptions: new() { ChatOptions = new() { Tools = [approvalTool] }, - PersistChatHistoryAtEndOfRun = false, }, callIndex: callIndex, capturedInputs: capturedInputs); @@ -260,7 +260,7 @@ public class ChatClientAgent_ApprovalsTests agentOptions: new() { ChatOptions = new() { Tools = [approvalTool] }, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }, callIndex: callIndex, capturedInputs: capturedInputs); diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ChatHistoryManagementTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ChatHistoryManagementTests.cs index 3e54dbc06e..83bdc2b2a2 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ChatHistoryManagementTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ChatHistoryManagementTests.cs @@ -520,7 +520,7 @@ public class ChatClientAgent_ChatHistoryManagementTests agentOptions: new() { ChatOptions = new() { Instructions = "Be helpful" }, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }, expectedServiceCallCount: 1, expectedHistory: @@ -554,7 +554,7 @@ public class ChatClientAgent_ChatHistoryManagementTests agentOptions: new() { ChatOptions = new() { Tools = [tool] }, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }, expectedServiceCallCount: 2, expectedHistory: @@ -583,7 +583,6 @@ public class ChatClientAgent_ChatHistoryManagementTests agentOptions: new() { ChatOptions = new() { Instructions = "Be helpful" }, - PersistChatHistoryAtEndOfRun = true, }, expectedServiceCallCount: 1, expectedHistory: @@ -615,7 +614,6 @@ public class ChatClientAgent_ChatHistoryManagementTests agentOptions: new() { ChatOptions = new() { Tools = [tool] }, - PersistChatHistoryAtEndOfRun = true, }, expectedServiceCallCount: 2, expectedHistory: @@ -644,7 +642,6 @@ public class ChatClientAgent_ChatHistoryManagementTests agentOptions: new() { ChatOptions = new() { Instructions = "Be helpful" }, - PersistChatHistoryAtEndOfRun = false, }, expectedServiceCallCount: 1); diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ChatOptionsMergingTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ChatOptionsMergingTests.cs index 28d38ea36a..e4df863ce0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ChatOptionsMergingTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatClientAgent_ChatOptionsMergingTests.cs @@ -176,12 +176,11 @@ public class ChatClientAgent_ChatOptionsMergingTests } /// - /// Verify that ChatOptions merging returns a non-null ChatOptions instance with null ConversationId - /// when both agent and request have no ChatOptions. The sentinel conversation ID is set for - /// per-service-call persistence and stripped before reaching the inner client. + /// Verify that when both agent and request have no ChatOptions, the inner client + /// receives null options. /// [Fact] - public async Task ChatOptionsMergingReturnsChatOptionsWithNullConversationIdWhenBothAgentAndRequestHaveNoneAsync() + public async Task ChatOptionsMergingReturnsNullChatOptionsWhenBothAgentAndRequestHaveNoneAsync() { // Arrange Mock mockService = new(); @@ -201,9 +200,8 @@ public class ChatClientAgent_ChatOptionsMergingTests // Act await agent.RunAsync(messages); - // Assert — ChatOptions is non-null because the sentinel was set, but ConversationId is null (stripped) - Assert.NotNull(capturedChatOptions); - Assert.Null(capturedChatOptions!.ConversationId); + // Assert + Assert.Null(capturedChatOptions); } /// diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatHistoryPersistingChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ServiceStoredSimulatingChatClientTests.cs similarity index 63% rename from dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatHistoryPersistingChatClientTests.cs rename to dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ServiceStoredSimulatingChatClientTests.cs index e7f91ab5d7..ea346dade9 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ChatHistoryPersistingChatClientTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/ChatClient/ServiceStoredSimulatingChatClientTests.cs @@ -13,15 +13,15 @@ using Moq.Protected; namespace Microsoft.Agents.AI.UnitTests; /// -/// Contains unit tests for the decorator, +/// Contains unit tests for the decorator, /// verifying that it persists messages via the after each /// individual service call by default, or marks messages for end-of-run persistence when the -/// option is enabled. +/// option is enabled. /// -public class ChatHistoryPersistingChatClientTests +public class ServiceStoredSimulatingChatClientTests { /// - /// Verifies that by default (PersistChatHistoryAtEndOfRun is false), + /// Verifies that by default (SimulateServiceStoredChatHistory is false), /// the ChatHistoryProvider receives messages after a successful non-streaming call. /// [Fact] @@ -50,7 +50,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { ChatHistoryProvider = mockChatHistoryProvider.Object, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act @@ -97,7 +97,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { ChatHistoryProvider = mockChatHistoryProvider.Object, - PersistChatHistoryAtEndOfRun = true, + SimulateServiceStoredChatHistory = true, }); // Act @@ -145,7 +145,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { ChatHistoryProvider = mockChatHistoryProvider.Object, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act @@ -163,11 +163,10 @@ public class ChatHistoryPersistingChatClientTests } /// - /// Verifies that the decorator is injected in persist mode by default - /// and can be discovered via GetService. + /// Verifies that the decorator is NOT injected by default (SimulateServiceStoredChatHistory is false). /// [Fact] - public void ChatClient_ContainsDecorator_InPersistMode_ByDefault() + public void ChatClient_DoesNotContainDecorator_ByDefault() { // Arrange Mock mockService = new(); @@ -176,16 +175,15 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new()); // Assert - var decorator = agent.ChatClient.GetService(); - Assert.NotNull(decorator); - Assert.False(decorator.MarkOnly); + var decorator = agent.ChatClient.GetService(); + Assert.Null(decorator); } /// - /// Verifies that the decorator is injected in mark-only mode when PersistChatHistoryAtEndOfRun is true. + /// Verifies that the decorator is injected when SimulateServiceStoredChatHistory is true. /// [Fact] - public void ChatClient_ContainsDecorator_InMarkOnlyMode_WhenPersistAtEndOfRun() + public void ChatClient_ContainsDecorator_WhenSimulateServiceStoredChatHistory() { // Arrange Mock mockService = new(); @@ -193,13 +191,12 @@ public class ChatHistoryPersistingChatClientTests // Act ChatClientAgent agent = new(mockService.Object, options: new() { - PersistChatHistoryAtEndOfRun = true, + SimulateServiceStoredChatHistory = true, }); // Assert - var decorator = agent.ChatClient.GetService(); + var decorator = agent.ChatClient.GetService(); Assert.NotNull(decorator); - Assert.True(decorator.MarkOnly); } /// @@ -218,27 +215,27 @@ public class ChatHistoryPersistingChatClientTests }); // Assert - var decorator = agent.ChatClient.GetService(); + var decorator = agent.ChatClient.GetService(); Assert.Null(decorator); } /// - /// Verifies that the PersistChatHistoryAtEndOfRun option is included in Clone(). + /// Verifies that the SimulateServiceStoredChatHistory option is included in Clone(). /// [Fact] - public void ChatClientAgentOptions_Clone_IncludesPersistChatHistoryAtEndOfRun() + public void ChatClientAgentOptions_Clone_IncludesSimulateServiceStoredChatHistory() { // Arrange var options = new ChatClientAgentOptions { - PersistChatHistoryAtEndOfRun = true, + SimulateServiceStoredChatHistory = true, }; // Act var cloned = options.Clone(); // Assert - Assert.True(cloned.PersistChatHistoryAtEndOfRun); + Assert.True(cloned.SimulateServiceStoredChatHistory); } /// @@ -292,7 +289,7 @@ public class ChatHistoryPersistingChatClientTests { ChatOptions = new() { Tools = [tool] }, ChatHistoryProvider = mockChatHistoryProvider.Object, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }, services: new ServiceCollection().BuildServiceProvider()); // Act @@ -361,7 +358,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { ChatHistoryProvider = mockChatHistoryProvider.Object, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act @@ -410,7 +407,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { AIContextProviders = [mockContextProvider.Object], - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act @@ -457,7 +454,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { AIContextProviders = [mockContextProvider.Object], - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act @@ -516,7 +513,7 @@ public class ChatHistoryPersistingChatClientTests { ChatHistoryProvider = mockChatHistoryProvider.Object, AIContextProviders = [mockContextProvider.Object], - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act @@ -590,7 +587,7 @@ public class ChatHistoryPersistingChatClientTests { ChatOptions = new() { Tools = [tool] }, ChatHistoryProvider = mockChatHistoryProvider.Object, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }, services: new ServiceCollection().BuildServiceProvider()); // Act @@ -655,7 +652,7 @@ public class ChatHistoryPersistingChatClientTests { ChatOptions = new() { Tools = [tool] }, ChatHistoryProvider = mockChatHistoryProvider.Object, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }, services: new ServiceCollection().BuildServiceProvider()); // Act @@ -680,52 +677,12 @@ public class ChatHistoryPersistingChatClientTests /// Verifies that after a successful run with per-service-call persistence, the notified /// messages are stamped with the persisted marker so they are not re-notified. /// - [Fact] - public async Task RunAsync_MarksNotifiedMessages_WithPersistedMarkerAsync() - { - // Arrange - Mock mockService = new(); - mockService.Setup( - s => s.GetResponseAsync( - It.IsAny>(), - It.IsAny(), - It.IsAny())).ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")])); - - Mock mockChatHistoryProvider = new(null, null, null); - mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]); - mockChatHistoryProvider - .Protected() - .Setup>>("InvokingCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) - .Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) => - new ValueTask>(ctx.RequestMessages.ToList())); - mockChatHistoryProvider - .Protected() - .Setup("InvokedCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) - .Returns(() => new ValueTask()); - - ChatClientAgent agent = new(mockService.Object, options: new() - { - ChatHistoryProvider = mockChatHistoryProvider.Object, - PersistChatHistoryAtEndOfRun = false, - }); - - // Act - var inputMessage = new ChatMessage(ChatRole.User, "test"); - var session = await agent.CreateSessionAsync() as ChatClientAgentSession; - await agent.RunAsync([inputMessage], session); - - // Assert — input message should be marked as persisted - Assert.True( - inputMessage.AdditionalProperties?.ContainsKey(ChatHistoryPersistingChatClient.PersistedMarkerKey) == true, - "Input message should be marked as persisted after a successful run."); - } - /// - /// Verifies that when per-service-call persistence is enabled and the inner client returns a - /// conversation ID, the session's ConversationId is updated after the service call. + /// Verifies that when the inner client returns a real conversation ID, + /// the session's ConversationId is updated after the run. /// [Fact] - public async Task RunAsync_UpdatesSessionConversationId_WhenPerServiceCallPersistenceEnabledAsync() + public async Task RunAsync_UpdatesSessionConversationId_WhenServiceReturnsOneAsync() { // Arrange const string ExpectedConversationId = "conv-123"; @@ -741,10 +698,7 @@ public class ChatHistoryPersistingChatClientTests ConversationId = ExpectedConversationId, }); - ChatClientAgent agent = new(mockService.Object, options: new() - { - PersistChatHistoryAtEndOfRun = false, - }); + ChatClientAgent agent = new(mockService.Object); // Act var session = await agent.CreateSessionAsync() as ChatClientAgentSession; @@ -766,8 +720,8 @@ public class ChatHistoryPersistingChatClientTests /// /// Verifies that when per-service-call persistence is active and no real conversation ID exists, - /// sets the - /// sentinel on the chat options and strips it before + /// sets the + /// sentinel on the chat options and strips it before /// forwarding to the inner client. /// [Fact] @@ -787,7 +741,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { ChatOptions = new() { Instructions = "test" }, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act @@ -819,7 +773,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { ChatOptions = new() { Instructions = "test" }, - PersistChatHistoryAtEndOfRun = true, + SimulateServiceStoredChatHistory = true, }); // Act @@ -854,7 +808,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Create a session with a real conversation ID. @@ -888,7 +842,7 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { ChatOptions = new() { Instructions = "test" }, - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act @@ -903,11 +857,12 @@ public class ChatHistoryPersistingChatClientTests } /// - /// Verifies that the session's conversation ID is NOT set to the sentinel after the run. - /// The sentinel should only exist transiently on the ChatOptions for the pipeline. + /// Verifies that the session's conversation ID IS set to the sentinel after the run + /// when simulating service-stored chat history. This allows subsequent runs to + /// skip provider resolution in the agent (the decorator handles it). /// [Fact] - public async Task RunAsync_SentinelDoesNotLeakToSession_WhenPerServiceCallPersistenceActiveAsync() + public async Task RunAsync_SetsSentinelOnSession_WhenSimulateServiceStoredChatHistoryActiveAsync() { // Arrange Mock mockService = new(); @@ -920,14 +875,440 @@ public class ChatHistoryPersistingChatClientTests ChatClientAgent agent = new(mockService.Object, options: new() { - PersistChatHistoryAtEndOfRun = false, + SimulateServiceStoredChatHistory = true, }); // Act var session = await agent.CreateSessionAsync() as ChatClientAgentSession; await agent.RunAsync([new(ChatRole.User, "test")], session); - // Assert — session should NOT have the sentinel conversation ID - Assert.Null(session!.ConversationId); + // Assert — session should have the sentinel conversation ID + Assert.Equal(ServiceStoredSimulatingChatClient.LocalHistoryConversationId, session!.ConversationId); + } + + /// + /// Verifies that when simulating service-stored chat history and the service returns + /// a real , the conflict detection in + /// throws because both a + /// and a service-managed ConversationId are present. + /// + [Fact] + public async Task RunAsync_Throws_WhenServiceReturnsRealConversationIdWithChatHistoryProviderAsync() + { + // Arrange + const string RealConversationId = "service-conv-456"; + + Mock mockChatHistoryProvider = new(null, null, null); + mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]); + mockChatHistoryProvider + .Protected() + .Setup>>("InvokingCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) => + new ValueTask>(ctx.RequestMessages.ToList())); + mockChatHistoryProvider + .Protected() + .Setup("InvokedCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns(new ValueTask()); + + Mock mockService = new(); + mockService.Setup( + s => s.GetResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]) + { + ConversationId = RealConversationId, + }); + + ChatClientAgent agent = new(mockService.Object, options: new() + { + ChatHistoryProvider = mockChatHistoryProvider.Object, + SimulateServiceStoredChatHistory = true, + }); + + // Act & Assert — conflict detection should throw + var session = await agent.CreateSessionAsync() as ChatClientAgentSession; + await Assert.ThrowsAsync(() => agent.RunAsync([new(ChatRole.User, "test")], session)); + } + + /// + /// Verifies that when simulating service-stored chat history and the request carries a real + /// , the decorator skips history loading but still + /// notifies s on success and updates the session ConversationId. + /// + [Fact] + public async Task RunAsync_NotifiesProvidersAndUpdatesSession_WhenRequestHasRealConversationIdAsync() + { + // Arrange + const string RealConversationId = "real-conv-request"; + const string ServiceConversationId = "real-conv-response"; + + Mock mockContextProvider = new(null, null, null); + mockContextProvider.SetupGet(p => p.StateKeys).Returns(["TestContextProvider"]); + mockContextProvider + .Protected() + .Setup>("InvokingCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((AIContextProvider.InvokingContext ctx, CancellationToken _) => + new ValueTask(new AIContext { Messages = ctx.AIContext.Messages })); + mockContextProvider + .Protected() + .Setup("InvokedCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns(new ValueTask()); + + Mock mockService = new(); + mockService.Setup( + s => s.GetResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]) + { + ConversationId = ServiceConversationId, + }); + + ChatClientAgent agent = new(mockService.Object, options: new() + { + SimulateServiceStoredChatHistory = true, + AIContextProviders = [mockContextProvider.Object], + }); + + // Create a session with a real conversation ID so it's on chatOptions. + var session = await agent.CreateSessionAsync(RealConversationId); + + // Act + await agent.RunAsync([new(ChatRole.User, "test")], session); + + // Assert — AIContextProvider.InvokedAsync should have been called + mockContextProvider + .Protected() + .Verify("InvokedCoreAsync", Times.Once(), + ItExpr.Is(x => + x.RequestMessages.Any(m => m.Text == "test") && + x.ResponseMessages!.Any(m => m.Text == "response")), + ItExpr.IsAny()); + + // Assert — session should have the service-returned ConversationId + Assert.Equal(ServiceConversationId, (session as ChatClientAgentSession)!.ConversationId); + } + + /// + /// Verifies that when simulating service-stored chat history and the request carries a real + /// , the decorator notifies providers of failure + /// when the inner client throws. + /// + [Fact] + public async Task RunAsync_NotifiesProvidersOfFailure_WhenRequestHasRealConversationIdAsync() + { + // Arrange + const string RealConversationId = "real-conv-failure"; + + Mock mockContextProvider = new(null, null, null); + mockContextProvider.SetupGet(p => p.StateKeys).Returns(["TestContextProvider"]); + mockContextProvider + .Protected() + .Setup>("InvokingCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((AIContextProvider.InvokingContext ctx, CancellationToken _) => + new ValueTask(new AIContext { Messages = ctx.AIContext.Messages })); + mockContextProvider + .Protected() + .Setup("InvokedCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns(new ValueTask()); + + Mock mockService = new(); + mockService.Setup( + s => s.GetResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new InvalidOperationException("Service error")); + + ChatClientAgent agent = new(mockService.Object, options: new() + { + SimulateServiceStoredChatHistory = true, + AIContextProviders = [mockContextProvider.Object], + }); + + var session = await agent.CreateSessionAsync(RealConversationId); + + // Act & Assert — should throw + await Assert.ThrowsAsync(() => agent.RunAsync([new(ChatRole.User, "test")], session)); + + // Assert — AIContextProvider.InvokedAsync should have been called with the failure + mockContextProvider + .Protected() + .Verify("InvokedCoreAsync", Times.Once(), + ItExpr.Is(x => x.InvokeException != null), + ItExpr.IsAny()); + } + + /// + /// Verifies that in the streaming path, when the request carries a real + /// , the decorator skips history loading but still + /// notifies providers and updates the session ConversationId. + /// + [Fact] + public async Task RunStreamingAsync_NotifiesProvidersAndUpdatesSession_WhenRequestHasRealConversationIdAsync() + { + // Arrange + const string RealConversationId = "real-conv-streaming"; + const string ServiceConversationId = "service-conv-streaming"; + + Mock mockContextProvider = new(null, null, null); + mockContextProvider.SetupGet(p => p.StateKeys).Returns(["TestContextProvider"]); + mockContextProvider + .Protected() + .Setup>("InvokingCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((AIContextProvider.InvokingContext ctx, CancellationToken _) => + new ValueTask(new AIContext { Messages = ctx.AIContext.Messages })); + mockContextProvider + .Protected() + .Setup("InvokedCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns(new ValueTask()); + + Mock mockService = new(); + mockService.Setup( + s => s.GetStreamingResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns(CreateAsyncEnumerableAsync( + new ChatResponseUpdate(ChatRole.Assistant, "streamed") { ConversationId = ServiceConversationId })); + + ChatClientAgent agent = new(mockService.Object, options: new() + { + SimulateServiceStoredChatHistory = true, + AIContextProviders = [mockContextProvider.Object], + }); + + var session = await agent.CreateSessionAsync(RealConversationId); + + // Act + await foreach (var _ in agent.RunStreamingAsync([new(ChatRole.User, "test")], session)) + { + // Consume all updates. + } + + // Assert — AIContextProvider.InvokedAsync should have been called + mockContextProvider + .Protected() + .Verify("InvokedCoreAsync", Times.Once(), + ItExpr.IsAny(), + ItExpr.IsAny()); + + // Assert — session should have the service-returned ConversationId + Assert.Equal(ServiceConversationId, (session as ChatClientAgentSession)!.ConversationId); + } + + /// + /// Verifies that when simulating and the service unexpectedly returns a real + /// (no ConversationId on the request), the decorator + /// notifies providers and updates the session ConversationId without setting the sentinel. + /// + [Fact] + public async Task RunAsync_NotifiesProvidersAndUpdatesSession_WhenServiceReturnsUnexpectedConversationIdAsync() + { + // Arrange + const string ServiceConversationId = "unexpected-conv-id"; + + Mock mockContextProvider = new(null, null, null); + mockContextProvider.SetupGet(p => p.StateKeys).Returns(["TestContextProvider"]); + mockContextProvider + .Protected() + .Setup>("InvokingCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((AIContextProvider.InvokingContext ctx, CancellationToken _) => + new ValueTask(new AIContext { Messages = ctx.AIContext.Messages })); + mockContextProvider + .Protected() + .Setup("InvokedCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns(new ValueTask()); + + Mock mockService = new(); + mockService.Setup( + s => s.GetResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")]) + { + ConversationId = ServiceConversationId, + }); + + // No ChatHistoryProvider — so conflict detection won't throw. + ChatClientAgent agent = new(mockService.Object, options: new() + { + SimulateServiceStoredChatHistory = true, + AIContextProviders = [mockContextProvider.Object], + }); + + // Act + var session = await agent.CreateSessionAsync() as ChatClientAgentSession; + await agent.RunAsync([new(ChatRole.User, "test")], session); + + // Assert — AIContextProvider.InvokedAsync should have been called + mockContextProvider + .Protected() + .Verify("InvokedCoreAsync", Times.Once(), + ItExpr.Is(x => + x.ResponseMessages!.Any(m => m.Text == "response")), + ItExpr.IsAny()); + + // Assert — session should have the service ConversationId, not the sentinel + Assert.Equal(ServiceConversationId, session!.ConversationId); + } + + /// + /// Verifies that in the streaming path, when the service returns a real ConversationId mid-stream + /// (no ConversationId on the request), the decorator notifies providers and updates the session. + /// + [Fact] + public async Task RunStreamingAsync_NotifiesProvidersAndUpdatesSession_WhenServiceReturnsUnexpectedConversationIdAsync() + { + // Arrange + const string ServiceConversationId = "unexpected-stream-conv"; + + Mock mockContextProvider = new(null, null, null); + mockContextProvider.SetupGet(p => p.StateKeys).Returns(["TestContextProvider"]); + mockContextProvider + .Protected() + .Setup>("InvokingCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((AIContextProvider.InvokingContext ctx, CancellationToken _) => + new ValueTask(new AIContext { Messages = ctx.AIContext.Messages })); + mockContextProvider + .Protected() + .Setup("InvokedCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns(new ValueTask()); + + Mock mockService = new(); + mockService.Setup( + s => s.GetStreamingResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns(CreateAsyncEnumerableAsync( + new ChatResponseUpdate(ChatRole.Assistant, "part1"), + new ChatResponseUpdate(null, "part2") { ConversationId = ServiceConversationId })); + + // No ChatHistoryProvider — so conflict detection won't throw. + ChatClientAgent agent = new(mockService.Object, options: new() + { + SimulateServiceStoredChatHistory = true, + AIContextProviders = [mockContextProvider.Object], + }); + + // Act + var session = await agent.CreateSessionAsync() as ChatClientAgentSession; + await foreach (var _ in agent.RunStreamingAsync([new(ChatRole.User, "test")], session)) + { + // Consume all updates. + } + + // Assert — AIContextProvider.InvokedAsync should have been called + mockContextProvider + .Protected() + .Verify("InvokedCoreAsync", Times.Once(), + ItExpr.IsAny(), + ItExpr.IsAny()); + + // Assert — session should have the service ConversationId, not the sentinel + Assert.Equal(ServiceConversationId, session!.ConversationId); + } + + /// + /// Verifies that when is true, + /// the decorator skips history loading and sentinel setting, letting the agent's + /// forced end-of-run path handle persistence. + /// + [Fact] + public async Task RunAsync_SkipsSimulation_WhenAllowBackgroundResponsesAsync() + { + // Arrange + IEnumerable? capturedMessages = null; + Mock mockService = new(); + mockService.Setup( + s => s.GetResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Callback, ChatOptions?, CancellationToken>((msgs, _, _) => capturedMessages = msgs) + .ReturnsAsync(new ChatResponse([new(ChatRole.Assistant, "response")])); + + Mock mockChatHistoryProvider = new(null, null, null); + mockChatHistoryProvider.SetupGet(p => p.StateKeys).Returns(["TestChatHistoryProvider"]); + mockChatHistoryProvider + .Protected() + .Setup>>("InvokingCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns((ChatHistoryProvider.InvokingContext ctx, CancellationToken _) => + { + // Add a history message to verify it's NOT prepended in this scenario. + var result = ctx.RequestMessages.ToList(); + result.Insert(0, new ChatMessage(ChatRole.Assistant, "history")); + return new ValueTask>(result); + }); + mockChatHistoryProvider + .Protected() + .Setup("InvokedCoreAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .Returns(new ValueTask()); + + ChatClientAgent agent = new(mockService.Object, options: new() + { + ChatHistoryProvider = mockChatHistoryProvider.Object, + SimulateServiceStoredChatHistory = true, + }); + + // Act + var session = await agent.CreateSessionAsync() as ChatClientAgentSession; + await agent.RunAsync( + [new(ChatRole.User, "test")], + session, + new AgentRunOptions { AllowBackgroundResponses = true }); + + // Assert — the inner client should NOT have received history messages + Assert.NotNull(capturedMessages); + var messageList = capturedMessages!.ToList(); + Assert.Single(messageList); + Assert.Equal("test", messageList[0].Text); + + // Assert — session should NOT have the sentinel (agent handles ConversationId at end-of-run) + Assert.NotEqual(ServiceStoredSimulatingChatClient.LocalHistoryConversationId, session!.ConversationId); + } + + /// + /// Verifies that in the streaming path, when is true, + /// the decorator skips history loading and sentinel setting. + /// + [Fact] + public async Task RunStreamingAsync_SkipsSimulation_WhenAllowBackgroundResponsesAsync() + { + // Arrange + Mock mockService = new(); + mockService.Setup( + s => s.GetStreamingResponseAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns(CreateAsyncEnumerableAsync(new ChatResponseUpdate(ChatRole.Assistant, "response"))); + + ChatClientAgent agent = new(mockService.Object, options: new() + { + SimulateServiceStoredChatHistory = true, + }); + + // Act + var session = await agent.CreateSessionAsync() as ChatClientAgentSession; + List updates = []; + await foreach (var update in agent.RunStreamingAsync( + [new(ChatRole.User, "test")], + session, + new AgentRunOptions { AllowBackgroundResponses = true })) + { + updates.Add(update); + } + + // Assert — updates should NOT carry the sentinel ConversationId + Assert.NotEmpty(updates); + + // Assert — session should NOT have the sentinel + Assert.NotEqual(ServiceStoredSimulatingChatClient.LocalHistoryConversationId, session!.ConversationId); } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/MessageMergerTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/MessageMergerTests.cs index 4181dad409..09d83966aa 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/MessageMergerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/MessageMergerTests.cs @@ -34,7 +34,9 @@ public class MessageMergerTests response.Messages[0].Role.Should().Be(ChatRole.Assistant); response.Messages[0].AuthorName.Should().Be(TestAuthorName1); response.AgentId.Should().Be(TestAgentId1); - response.CreatedAt.Should().NotBe(creationTime); + response.CreatedAt.Should().HaveValue(); + response.CreatedAt.Value.Should().BeOnOrAfter(creationTime); + response.CreatedAt.Value.Should().BeCloseTo(creationTime, precision: TimeSpan.FromSeconds(5)); response.Messages[0].CreatedAt.Should().Be(creationTime); response.Messages[0].Contents.Should().HaveCount(1); response.FinishReason.Should().BeNull(); diff --git a/dotnet/wf-code-gen-impact.md b/dotnet/wf-code-gen-impact.md deleted file mode 100644 index b49c8c0594..0000000000 --- a/dotnet/wf-code-gen-impact.md +++ /dev/null @@ -1,257 +0,0 @@ -# Source Generator for Workflow Executors: Rationale and Impact - -## Overview - -The Microsoft Agents AI Workflows framework has introduced a Roslyn source generator (`Microsoft.Agents.AI.Workflows.Generators`) that replaces the previous reflection-based approach for discovering and registering message handlers. This document explains why this change was made, what benefits it provides, and how it impacts framework users. - -## Why Move from Reflection to Code Generation? - -### The Previous Approach: `ReflectingExecutor` - -Previously, executors that needed automatic handler discovery inherited from `ReflectingExecutor` and implemented marker interfaces like `IMessageHandler`: - -```csharp -// Old approach - reflection-based -public class MyExecutor : ReflectingExecutor, - IMessageHandler, - IMessageHandler -{ - public ValueTask HandleAsync(QueryMessage msg, IWorkflowContext ctx, CancellationToken ct) - { - // Handle query - } - - public ValueTask HandleAsync(CommandMessage msg, IWorkflowContext ctx, CancellationToken ct) - { - // Handle command and return result - } -} -``` - -This approach had several limitations: - -1. **Runtime overhead**: Handler discovery happened at runtime via reflection, adding latency to executor initialization -2. **No AOT compatibility**: Reflection-based discovery doesn't work with Native AOT compilation -3. **Redundant declarations**: The interface list duplicated information already present in method signatures -4. **Limited metadata**: No clean way to declare yield/send types for protocol validation -5. **Hidden errors**: Invalid handler signatures weren't caught until runtime - -### The New Approach: `[MessageHandler]` Attribute - -The source generator enables a cleaner, attribute-based pattern: - -```csharp -// New approach - source generated -[SendsMessage(typeof(PollToken))] -public partial class MyExecutor : Executor -{ - [MessageHandler] - private ValueTask HandleQueryAsync(QueryMessage msg, IWorkflowContext ctx, CancellationToken ct) - { - // Handle query - } - - [MessageHandler(Yield = [typeof(StreamChunk)], Send = [typeof(InternalMessage)])] - private ValueTask HandleCommandAsync(CommandMessage msg, IWorkflowContext ctx, CancellationToken ct) - { - // Handle command and return result - } -} -``` - -The generator produces a partial class with `ConfigureRoutes()`, `ConfigureSentTypes()`, and `ConfigureYieldTypes()` implementations at compile time. - -## What's Better About Code Generation? - -### 1. Compile-Time Validation - -Invalid handler signatures are caught during compilation, not at runtime: - -```csharp -[MessageHandler] -private void InvalidHandler(string msg) // Error WFGEN005: Missing IWorkflowContext parameter -{ -} -``` - -Diagnostic errors include: -- `WFGEN001`: Handler missing `IWorkflowContext` parameter -- `WFGEN002`: Invalid return type (must be `void`, `ValueTask`, or `ValueTask`) -- `WFGEN003`: Executor class must be `partial` -- `WFGEN004`: `[MessageHandler]` on non-Executor class -- `WFGEN005`: Insufficient parameters -- `WFGEN006`: `ConfigureRoutes` already manually defined - -### 2. Zero Runtime Reflection - -All handler registration happens at compile time. The generated code is simple, direct method calls: - -```csharp -// Generated code -protected override RouteBuilder ConfigureRoutes(RouteBuilder routeBuilder) -{ - return routeBuilder - .AddHandler(this.HandleQueryAsync) - .AddHandler(this.HandleCommandAsync); -} -``` - -This eliminates: -- Reflection overhead during initialization -- Assembly scanning -- Dynamic delegate creation - -### 3. Native AOT Compatibility - -Because there's no runtime reflection, executors work seamlessly with .NET Native AOT compilation. This enables: -- Faster startup times -- Smaller deployment sizes -- Deployment to environments that don't support JIT compilation - -### 4. Explicit Protocol Metadata - -The `Yield` and `Send` properties on `[MessageHandler]` plus class-level `[SendsMessage]` and `[YieldsMessage]` attributes provide explicit protocol documentation: - -```csharp -[SendsMessage(typeof(PollToken))] // This executor sends PollToken messages -[YieldsMessage(typeof(FinalResult))] // This executor yields FinalResult to workflow output -public partial class MyExecutor : Executor -{ - [MessageHandler( - Yield = [typeof(StreamChunk)], // This handler yields StreamChunk - Send = [typeof(InternalQuery)])] // This handler sends InternalQuery - private ValueTask HandleAsync(Request req, IWorkflowContext ctx) { ... } -} -``` - -This metadata enables: -- Static protocol validation -- Better IDE tooling and documentation -- Clearer code intent - -### 5. Handler Accessibility Freedom - -Handlers can be `private`, `protected`, `internal`, or `public`. The old interface-based approach required public methods. Now you can encapsulate handler implementations: - -```csharp -public partial class MyExecutor : Executor -{ - [MessageHandler] - private ValueTask HandleInternalAsync(InternalMessage msg, IWorkflowContext ctx) - { - // Private handler - implementation detail - } -} -``` - -### 6. Cleaner Inheritance - -The generator properly handles inheritance chains, calling `base.ConfigureRoutes()` when appropriate: - -```csharp -public partial class DerivedExecutor : BaseExecutor -{ - [MessageHandler] - private ValueTask HandleDerivedAsync(DerivedMessage msg, IWorkflowContext ctx) { ... } -} - -// Generated: -protected override RouteBuilder ConfigureRoutes(RouteBuilder routeBuilder) -{ - routeBuilder = base.ConfigureRoutes(routeBuilder); // Preserves base handlers - return routeBuilder - .AddHandler(this.HandleDerivedAsync); -} -``` - -## New Capabilities Enabled - -### 1. Static Workflow Analysis - -With explicit yield/send metadata, tools can analyze workflow graphs at compile time: -- Validate that all message types have handlers -- Detect unreachable executors -- Generate workflow documentation - -### 2. Trimming-Safe Deployments - -The generated code contains no reflection, making it fully compatible with IL trimming. This reduces deployment size significantly for serverless and edge scenarios. - -### 3. Better IDE Experience - -Because the generator runs in the IDE, you get: -- Immediate feedback on handler signature errors -- IntelliSense for generated methods -- Go-to-definition on generated code - -### 4. Protocol Documentation Generation - -The explicit type metadata can be used to generate: -- API documentation -- OpenAPI/Swagger specs for workflow endpoints -- Visual workflow diagrams - -## Impact on Framework Users - -### Migration Path - -Existing code using `ReflectingExecutor` continues to work but is marked `[Obsolete]`. To migrate: - -1. Change base class from `ReflectingExecutor` to `Executor` -2. Add `partial` modifier to the class -3. Replace `IMessageHandler` interfaces with `[MessageHandler]` attributes -4. Optionally add `Yield`/`Send` metadata for protocol validation - -**Before:** -```csharp -public class MyExecutor : ReflectingExecutor, IMessageHandler -{ - public ValueTask HandleAsync(Query q, IWorkflowContext ctx, CancellationToken ct) { ... } -} -``` - -**After:** -```csharp -public partial class MyExecutor : Executor -{ - [MessageHandler] - private ValueTask HandleQueryAsync(Query q, IWorkflowContext ctx, CancellationToken ct) { ... } -} -``` - -### Breaking Changes - -- Classes using `[MessageHandler]` **must** be `partial` -- Handler methods must have at least 2 parameters: `(TMessage, IWorkflowContext)` -- Return type must be `void`, `ValueTask`, or `ValueTask` - -### Performance Improvements - -Users can expect: -- **Faster executor initialization**: No reflection overhead -- **Reduced memory allocation**: No dynamic delegate creation -- **AOT deployment support**: Full Native AOT compatibility -- **Smaller trimmed deployments**: No reflection metadata preserved - -### NuGet Package - -The generator is distributed as a separate NuGet package (`Microsoft.Agents.AI.Workflows.Generators`) that's automatically referenced by the main Workflows package. It's packaged as an analyzer, so it: -- Runs automatically during build -- Requires no additional configuration -- Works in all IDEs that support Roslyn analyzers - -## Summary - -The move from reflection to source generation represents a significant improvement in the Workflows framework: - -| Aspect | Reflection (Old) | Source Generator (New) | -|--------|------------------|------------------------| -| Handler discovery | Runtime | Compile-time | -| Error detection | Runtime exceptions | Compiler errors | -| AOT support | No | Yes | -| Trimming support | Limited | Full | -| Protocol metadata | Implicit | Explicit | -| Handler visibility | Public only | Any | -| Initialization speed | Slower | Faster | - -The source generator approach aligns with modern .NET best practices and positions the framework for future scenarios including edge computing, serverless, and mobile deployments where AOT compilation and minimal footprint are essential. diff --git a/dotnet/wf-source-gen-bp.md b/dotnet/wf-source-gen-bp.md deleted file mode 100644 index c0f3d25892..0000000000 --- a/dotnet/wf-source-gen-bp.md +++ /dev/null @@ -1,439 +0,0 @@ -# Source Generator Best Practices Review - -This document reviews the Workflow Executor Route Source Generator implementation against the official Roslyn Source Generator Cookbook best practices from the dotnet/roslyn repository. - -## Reference Documentation - -- [Source Generators Cookbook](https://github.com/dotnet/roslyn/blob/main/docs/features/source-generators.cookbook.md) -- [Incremental Generators Cookbook](https://github.com/dotnet/roslyn/blob/main/docs/features/incremental-generators.cookbook.md) - ---- - -## Executive Summary - -| Category | Status | Priority | -|----------|--------|----------| -| Generator Type | PASS | - | -| Attribute-Based Detection | FAIL | HIGH | -| Model Value Equality | FAIL | HIGH | -| Collection Equality | FAIL | HIGH | -| Symbol/SyntaxNode Storage | PASS | - | -| Code Generation Approach | PASS | - | -| Diagnostics | PASS | - | -| Pipeline Efficiency | FAIL | MEDIUM | -| CancellationToken Handling | PARTIAL | LOW | - -**Overall Assessment**: The generator follows several best practices but has critical performance issues that should be addressed before production use. The most significant issue is not using `ForAttributeWithMetadataName`, which the Roslyn team states is "at least 99x more efficient" than `CreateSyntaxProvider`. - ---- - -## Detailed Analysis - -### 1. Generator Interface Selection - -**Best Practice**: Use `IIncrementalGenerator` instead of the deprecated `ISourceGenerator`. - -**Our Implementation**: PASS - -```csharp -// ExecutorRouteGenerator.cs:19 -public sealed class ExecutorRouteGenerator : IIncrementalGenerator -``` - -The generator correctly implements `IIncrementalGenerator`, the recommended interface for new generators. - ---- - -### 2. Attribute-Based Detection with ForAttributeWithMetadataName - -**Best Practice**: Use `ForAttributeWithMetadataName()` for attribute-based discovery. - -> "This utility method is at least 99x more efficient than `SyntaxProvider.CreateSyntaxProvider`, and in many cases even more efficient." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: FAIL (HIGH PRIORITY) - -```csharp -// ExecutorRouteGenerator.cs:25-30 -var executorCandidates = context.SyntaxProvider - .CreateSyntaxProvider( - predicate: static (node, _) => SyntaxDetector.IsExecutorCandidate(node), - transform: static (ctx, ct) => SemanticAnalyzer.Analyze(ctx, ct, out _)) -``` - -**Problem**: We use `CreateSyntaxProvider` with manual attribute detection in `SyntaxDetector`. This requires the generator to examine every syntax node in the compilation, whereas `ForAttributeWithMetadataName` uses the compiler's built-in attribute index for O(1) lookup. - -**Recommended Fix**: - -```csharp -var executorCandidates = context.SyntaxProvider - .ForAttributeWithMetadataName( - fullyQualifiedMetadataName: "Microsoft.Agents.AI.Workflows.MessageHandlerAttribute", - predicate: static (node, _) => node is MethodDeclarationSyntax, - transform: static (ctx, ct) => AnalyzeMethodWithAttribute(ctx, ct)) - .Collect() - .SelectMany((methods, _) => GroupByContainingClass(methods)); -``` - -**Impact**: Current approach causes IDE lag on every keystroke in large projects. - ---- - -### 3. Model Value Equality (Records vs Classes) - -**Best Practice**: Use `record` types for pipeline models to get automatic value equality. - -> "Use `record`s, rather than `class`es, so that value equality is generated for you." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: FAIL (HIGH PRIORITY) - -```csharp -// HandlerInfo.cs:28 -internal sealed class HandlerInfo { ... } - -// ExecutorInfo.cs:10 -internal sealed class ExecutorInfo { ... } -``` - -**Problem**: Both `HandlerInfo` and `ExecutorInfo` are `sealed class` types, which use reference equality by default. The incremental generator caches results based on equality comparison—when the model equals the previous run's model, regeneration is skipped. With reference equality, every analysis produces a "new" object, defeating caching entirely. - -**Recommended Fix**: - -```csharp -// HandlerInfo.cs -internal sealed record HandlerInfo( - string MethodName, - string InputTypeName, - string? OutputTypeName, - HandlerSignatureKind SignatureKind, - bool HasCancellationToken, - EquatableArray? YieldTypes, - EquatableArray? SendTypes); - -// ExecutorInfo.cs -internal sealed record ExecutorInfo( - string? Namespace, - string ClassName, - string? GenericParameters, - bool IsNested, - string ContainingTypeChain, - bool BaseHasConfigureRoutes, - EquatableArray Handlers, - EquatableArray ClassSendTypes, - EquatableArray ClassYieldTypes); -``` - -**Impact**: Without value equality, the generator regenerates code on every compilation even when nothing changed. - ---- - -### 4. Collection Equality - -**Best Practice**: Use custom equatable wrappers for collections since `ImmutableArray` uses reference equality. - -> "Arrays, `ImmutableArray`, and `List` use reference equality by default. Wrap collections with custom types implementing value-based equality." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: FAIL (HIGH PRIORITY) - -```csharp -// ExecutorInfo.cs:46 -public ImmutableArray Handlers { get; } - -// HandlerInfo.cs:58-63 -public ImmutableArray? YieldTypes { get; } -public ImmutableArray? SendTypes { get; } -``` - -**Problem**: `ImmutableArray` compares by reference, not by contents. Two arrays with identical elements are considered unequal, breaking incremental caching. - -**Recommended Fix**: Create an `EquatableArray` wrapper: - -```csharp -internal readonly struct EquatableArray : IEquatable>, IEnumerable - where T : IEquatable -{ - private readonly ImmutableArray _array; - - public EquatableArray(ImmutableArray array) => _array = array; - - public bool Equals(EquatableArray other) - { - if (_array.Length != other._array.Length) return false; - for (int i = 0; i < _array.Length; i++) - { - if (!_array[i].Equals(other._array[i])) return false; - } - return true; - } - - public override int GetHashCode() - { - var hash = new HashCode(); - foreach (var item in _array) hash.Add(item); - return hash.ToHashCode(); - } - - // ... IEnumerable implementation -} -``` - -**Impact**: Same as model equality—caching is completely broken for handlers and type arrays. - ---- - -### 5. Symbol and SyntaxNode Storage - -**Best Practice**: Never store `ISymbol` or `SyntaxNode` in pipeline models. - -> "Storing `ISymbol` references blocks garbage collection and roots old compilations unnecessarily. Extract only the information you need—typically string representations work well—into your equatable models." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: PASS - -The models correctly store only primitive types and strings: - -```csharp -// HandlerInfo.cs - stores strings, not symbols -public string MethodName { get; } -public string InputTypeName { get; } -public string? OutputTypeName { get; } - -// ExecutorInfo.cs - stores strings, not symbols -public string? Namespace { get; } -public string ClassName { get; } -``` - -The `SemanticAnalyzer` correctly extracts string representations from symbols: - -```csharp -// SemanticAnalyzer.cs:300-301 -var inputType = methodSymbol.Parameters[0].Type; -var inputTypeName = inputType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); -``` - ---- - -### 6. Code Generation Approach - -**Best Practice**: Use `StringBuilder` for code generation, not `SyntaxNode` construction. - -> "Avoid constructing `SyntaxNode`s for output; they're complex to format correctly and `NormalizeWhitespace()` is expensive. Instead, use a `StringBuilder` wrapper that tracks indentation levels." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: PASS - -```csharp -// SourceBuilder.cs:17-19 -public static string Generate(ExecutorInfo info) -{ - var sb = new StringBuilder(); -``` - -The `SourceBuilder` correctly uses `StringBuilder` with manual indentation tracking. - ---- - -### 7. Diagnostic Reporting - -**Best Practice**: Use `ReportDiagnostic` for surfacing issues to users. - -**Our Implementation**: PASS - -```csharp -// ExecutorRouteGenerator.cs:44-50 -context.RegisterSourceOutput(diagnosticsProvider, static (ctx, diagnostics) => -{ - foreach (var diagnostic in diagnostics) - { - ctx.ReportDiagnostic(diagnostic); - } -}); -``` - -Diagnostics are well-defined with appropriate severities: - -| ID | Severity | Description | -|----|----------|-------------| -| WFGEN001 | Error | Missing IWorkflowContext parameter | -| WFGEN002 | Error | Invalid return type | -| WFGEN003 | Error | Class must be partial | -| WFGEN004 | Warning | Not an Executor | -| WFGEN005 | Error | Insufficient parameters | -| WFGEN006 | Info | ConfigureRoutes already defined | -| WFGEN007 | Error | Handler cannot be static | - ---- - -### 8. Pipeline Efficiency - -**Best Practice**: Avoid duplicate work in the pipeline. - -**Our Implementation**: FAIL (MEDIUM PRIORITY) - -```csharp -// ExecutorRouteGenerator.cs:25-41 -// Pipeline 1: Get executor candidates -var executorCandidates = context.SyntaxProvider - .CreateSyntaxProvider( - predicate: static (node, _) => SyntaxDetector.IsExecutorCandidate(node), - transform: static (ctx, ct) => SemanticAnalyzer.Analyze(ctx, ct, out _)) - ... - -// Pipeline 2: Get diagnostics (duplicates the same work!) -var diagnosticsProvider = context.SyntaxProvider - .CreateSyntaxProvider( - predicate: static (node, _) => SyntaxDetector.IsExecutorCandidate(node), - transform: static (ctx, ct) => - { - SemanticAnalyzer.Analyze(ctx, ct, out var diagnostics); - return diagnostics; - }) -``` - -**Problem**: The same syntax detection and semantic analysis runs twice—once for extracting `ExecutorInfo` and once for extracting diagnostics. - -**Recommended Fix**: Return both in a single pipeline: - -```csharp -var analysisResults = context.SyntaxProvider - .ForAttributeWithMetadataName(...) - .Select((ctx, ct) => { - var info = SemanticAnalyzer.Analyze(ctx, ct, out var diagnostics); - return (Info: info, Diagnostics: diagnostics); - }); - -// Split for different outputs -context.RegisterSourceOutput( - analysisResults.Where(r => r.Info != null).Select((r, _) => r.Info!), - GenerateSource); - -context.RegisterSourceOutput( - analysisResults.Where(r => r.Diagnostics.Length > 0).Select((r, _) => r.Diagnostics), - ReportDiagnostics); -``` - ---- - -### 9. Base Type Chain Scanning - -**Best Practice**: Avoid scanning indirect type relationships when possible. - -> "Never scan for types that indirectly implement interfaces, inherit from base types, or acquire attributes through inheritance hierarchies. This pattern forces the generator to inspect every type's `AllInterfaces` or base-type chain on every keystroke." -> — Roslyn Incremental Generators Cookbook - -**Our Implementation**: PARTIAL CONCERN - -```csharp -// SemanticAnalyzer.cs:126-141 -private static bool DerivesFromExecutor(INamedTypeSymbol classSymbol) -{ - var current = classSymbol.BaseType; - while (current != null) - { - var fullName = current.OriginalDefinition.ToDisplayString(); - if (fullName == ExecutorTypeName || fullName.StartsWith(ExecutorTypeName + "<", ...)) - { - return true; - } - current = current.BaseType; - } - return false; -} -``` - -**Analysis**: We do walk the base type chain, but this only happens after attribute filtering (classes must have `[MessageHandler]` methods). Since this is targeted to specific candidates rather than scanning all types, the performance impact is acceptable. However, if we switch to `ForAttributeWithMetadataName`, the attribute is on methods, so we'd need to check the containing class's base types—which is still targeted. - ---- - -### 10. CancellationToken Handling - -**Best Practice**: Respect `CancellationToken` in long-running operations. - -**Our Implementation**: PARTIAL (LOW PRIORITY) - -The `CancellationToken` is passed through to semantic model calls: - -```csharp -// SemanticAnalyzer.cs:46 -var classSymbol = semanticModel.GetDeclaredSymbol(classDecl, cancellationToken); -``` - -However, there are no explicit `cancellationToken.ThrowIfCancellationRequested()` calls in loops like `AnalyzeHandlers`. For most compilations this is fine, but very large classes with many handlers might benefit from periodic checks. - ---- - -### 11. File Naming Convention - -**Best Practice**: Use descriptive generated file names with `.g.cs` suffix. - -**Our Implementation**: PASS - -```csharp -// ExecutorRouteGenerator.cs:62-91 -private static string GetHintName(ExecutorInfo info) -{ - // Produces: "Namespace.ClassName.g.cs" or "Namespace.Outer.Inner.ClassName.g.cs" - ... - sb.Append(".g.cs"); - return sb.ToString(); -} -``` - ---- - -## Recommended Action Plan - -### High Priority (Performance Critical) - -1. **Switch to `ForAttributeWithMetadataName`** - - Estimated impact: 99x+ performance improvement for attribute detection - - Requires restructuring the pipeline to collect methods then group by class - -2. **Convert models to records** - - Change `HandlerInfo` and `ExecutorInfo` from `sealed class` to `sealed record` - - Enables automatic value equality for incremental caching - -3. **Implement `EquatableArray`** - - Create wrapper struct with value-based equality - - Replace all `ImmutableArray` usages in models - -### Medium Priority (Efficiency) - -4. **Eliminate duplicate pipeline execution** - - Combine info extraction and diagnostic collection into single pipeline - - Split outputs using `Where` and `Select` - -### Low Priority (Polish) - -5. **Add periodic cancellation checks** - - Add `ThrowIfCancellationRequested()` in handler analysis loop - - Only needed for extremely large classes - ---- - -## Compliance Matrix - -| Best Practice | Cookbook Reference | Status | Fix Required | -|--------------|-------------------|--------|--------------| -| Use IIncrementalGenerator | Main cookbook | PASS | No | -| Use ForAttributeWithMetadataName | Incremental cookbook | FAIL | Yes (High) | -| Use records for models | Incremental cookbook | FAIL | Yes (High) | -| Implement collection equality | Incremental cookbook | FAIL | Yes (High) | -| Don't store ISymbol/SyntaxNode | Incremental cookbook | PASS | No | -| Use StringBuilder for codegen | Incremental cookbook | PASS | No | -| Report diagnostics properly | Main cookbook | PASS | No | -| Avoid duplicate pipeline work | Incremental cookbook | FAIL | Yes (Medium) | -| Respect CancellationToken | Main cookbook | PARTIAL | Optional | -| Use .g.cs file suffix | Main cookbook | PASS | No | -| Additive-only generation | Main cookbook | PASS | No | -| No language feature emulation | Main cookbook | PASS | No | - ---- - -## Conclusion - -The source generator implementation demonstrates solid understanding of Roslyn generator fundamentals—correct interface usage, proper diagnostic reporting, and appropriate code generation patterns. However, critical performance optimizations are missing that could cause significant IDE lag in production environments. - -The three high-priority fixes (ForAttributeWithMetadataName, record models, and EquatableArray) should be implemented before the generator is used in large codebases. These changes will enable proper incremental caching, reducing regeneration from "every keystroke" to "only when relevant code changes." diff --git a/dotnet/wf-source-gen-changes.md b/dotnet/wf-source-gen-changes.md deleted file mode 100644 index cc0aca5157..0000000000 --- a/dotnet/wf-source-gen-changes.md +++ /dev/null @@ -1,258 +0,0 @@ -# Workflow Executor Route Source Generator - Implementation Summary - -This document summarizes all changes made to implement a Roslyn source generator that replaces the reflection-based `ReflectingExecutor` pattern with compile-time code generation using `[MessageHandler]` attributes. - -## Overview - -The source generator automatically discovers methods marked with `[MessageHandler]` and generates `ConfigureRoutes`, `ConfigureSentTypes`, and `ConfigureYieldTypes` method implementations at compile time. This improves AOT compatibility and eliminates the need for the CRTP (Curiously Recurring Template Pattern) used by `ReflectingExecutor`. - -## New Files Created - -### Attributes (3 files) - -| File | Purpose | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows/Attributes/MessageHandlerAttribute.cs` | Marks methods as message handlers with optional `Yield` and `Send` type arrays | -| `src/Microsoft.Agents.AI.Workflows/Attributes/SendsMessageAttribute.cs` | Class-level attribute declaring message types an executor may send | -| `src/Microsoft.Agents.AI.Workflows/Attributes/YieldsMessageAttribute.cs` | Class-level attribute declaring output types an executor may yield | - -### Source Generator Project (8 files) - -| File | Purpose | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows.Generators/Microsoft.Agents.AI.Workflows.Generators.csproj` | Project file targeting netstandard2.0 with Roslyn component settings | -| `src/Microsoft.Agents.AI.Workflows.Generators/ExecutorRouteGenerator.cs` | Main incremental generator implementing `IIncrementalGenerator` | -| `src/Microsoft.Agents.AI.Workflows.Generators/Models/HandlerInfo.cs` | Data model for handler method information | -| `src/Microsoft.Agents.AI.Workflows.Generators/Models/ExecutorInfo.cs` | Data model for executor class information | -| `src/Microsoft.Agents.AI.Workflows.Generators/Analysis/SyntaxDetector.cs` | Fast syntax-level candidate detection | -| `src/Microsoft.Agents.AI.Workflows.Generators/Analysis/SemanticAnalyzer.cs` | Semantic validation and type extraction | -| `src/Microsoft.Agents.AI.Workflows.Generators/Generation/SourceBuilder.cs` | Code generation logic | -| `src/Microsoft.Agents.AI.Workflows.Generators/Diagnostics/DiagnosticDescriptors.cs` | Analyzer diagnostic definitions | - -## Files Modified - -### Project Files - -| File | Changes | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows/Microsoft.Agents.AI.Workflows.csproj` | Added generator project reference and `InternalsVisibleTo` for generator tests | -| `Directory.Packages.props` | Added `Microsoft.CodeAnalysis.Analyzers` version 3.11.0 | -| `agent-framework-dotnet.slnx` | Added generator project to solution | - -### Obsolete Annotations - -| File | Changes | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows/Reflection/ReflectingExecutor.cs` | Added `[Obsolete]` attribute with migration guidance | -| `src/Microsoft.Agents.AI.Workflows/Reflection/IMessageHandler.cs` | Added `[Obsolete]` to both `IMessageHandler` and `IMessageHandler` interfaces | - -### Pragma Suppressions for Internal Obsolete Usage - -| File | Changes | -|------|---------| -| `src/Microsoft.Agents.AI.Workflows/Executor.cs` | Added `#pragma warning disable CS0618` | -| `src/Microsoft.Agents.AI.Workflows/StatefulExecutor.cs` | Added `#pragma warning disable CS0618` | -| `src/Microsoft.Agents.AI.Workflows/Reflection/RouteBuilderExtensions.cs` | Added `#pragma warning disable CS0618` | -| `src/Microsoft.Agents.AI.Workflows/Reflection/MessageHandlerInfo.cs` | Added `#pragma warning disable CS0618` | - -### Test File Pragma Suppressions - -| File | Changes | -|------|---------| -| `tests/Microsoft.Agents.AI.Workflows.UnitTests/Sample/01_Simple_Workflow_Sequential.cs` | Added `#pragma warning disable CS0618` for legacy pattern testing | -| `tests/Microsoft.Agents.AI.Workflows.UnitTests/Sample/02_Simple_Workflow_Condition.cs` | Added `#pragma warning disable CS0618` for legacy pattern testing | -| `tests/Microsoft.Agents.AI.Workflows.UnitTests/Sample/03_Simple_Workflow_Loop.cs` | Added `#pragma warning disable CS0618` for legacy pattern testing | -| `tests/Microsoft.Agents.AI.Workflows.UnitTests/ReflectionSmokeTest.cs` | Added `#pragma warning disable CS0618` for legacy pattern testing | - -## Attribute Definitions - -### MessageHandlerAttribute - -```csharp -[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = false)] -public sealed class MessageHandlerAttribute : Attribute -{ - public Type[]? Yield { get; set; } // Types yielded as workflow outputs - public Type[]? Send { get; set; } // Types sent to other executors -} -``` - -### SendsMessageAttribute - -```csharp -[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = true)] -public sealed class SendsMessageAttribute : Attribute -{ - public Type Type { get; } - public SendsMessageAttribute(Type type) => this.Type = Throw.IfNull(type); -} -``` - -### YieldsMessageAttribute - -```csharp -[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = true)] -public sealed class YieldsMessageAttribute : Attribute -{ - public Type Type { get; } - public YieldsMessageAttribute(Type type) => this.Type = Throw.IfNull(type); -} -``` - -## Diagnostic Rules - -| ID | Severity | Description | -|----|----------|-------------| -| `WFGEN001` | Error | Handler method must have at least 2 parameters (message and IWorkflowContext) | -| `WFGEN002` | Error | Handler method's second parameter must be IWorkflowContext | -| `WFGEN003` | Error | Handler method must return void, ValueTask, or ValueTask | -| `WFGEN004` | Error | Executor class with [MessageHandler] methods must be declared as partial | -| `WFGEN005` | Warning | [MessageHandler] attribute on method in non-Executor class (ignored) | -| `WFGEN006` | Info | ConfigureRoutes already defined manually, [MessageHandler] methods ignored | -| `WFGEN007` | Error | Handler method's third parameter (if present) must be CancellationToken | - -## Handler Signature Support - -The generator supports the following method signatures: - -| Return Type | Parameters | Generated Call | -|-------------|------------|----------------| -| `void` | `(TMessage, IWorkflowContext)` | `AddHandler(this.Method)` | -| `void` | `(TMessage, IWorkflowContext, CancellationToken)` | `AddHandler(this.Method)` | -| `ValueTask` | `(TMessage, IWorkflowContext)` | `AddHandler(this.Method)` | -| `ValueTask` | `(TMessage, IWorkflowContext, CancellationToken)` | `AddHandler(this.Method)` | -| `TResult` | `(TMessage, IWorkflowContext)` | `AddHandler(this.Method)` | -| `TResult` | `(TMessage, IWorkflowContext, CancellationToken)` | `AddHandler(this.Method)` | -| `ValueTask` | `(TMessage, IWorkflowContext)` | `AddHandler(this.Method)` | -| `ValueTask` | `(TMessage, IWorkflowContext, CancellationToken)` | `AddHandler(this.Method)` | - -## Generated Code Example - -### Input (User Code) - -```csharp -[SendsMessage(typeof(PollToken))] -public partial class MyChatExecutor : Executor -{ - [MessageHandler] - private async ValueTask HandleQueryAsync( - ChatQuery query, IWorkflowContext ctx, CancellationToken ct) - { - return new ChatResponse(...); - } - - [MessageHandler(Yield = new[] { typeof(StreamChunk) }, Send = new[] { typeof(InternalMessage) })] - private void HandleStream(StreamRequest req, IWorkflowContext ctx) - { - // Handler implementation - } -} -``` - -### Output (Generated Code) - -```csharp -// -#nullable enable - -namespace MyNamespace; - -partial class MyChatExecutor -{ - protected override RouteBuilder ConfigureRoutes(RouteBuilder routeBuilder) - { - return routeBuilder - .AddHandler(this.HandleQueryAsync) - .AddHandler(this.HandleStream); - } - - protected override ISet ConfigureSentTypes() - { - var types = base.ConfigureSentTypes(); - types.Add(typeof(PollToken)); - types.Add(typeof(InternalMessage)); - return types; - } - - protected override ISet ConfigureYieldTypes() - { - var types = base.ConfigureYieldTypes(); - types.Add(typeof(ChatResponse)); - types.Add(typeof(StreamChunk)); - return types; - } -} -``` - -## Build Issues Resolved - -### 1. NU1008 - Central Package Management -Package references in the generator project had inline versions, which conflicts with central package management. Fixed by removing `Version` attributes from `PackageReference` items. - -### 2. RS2008 - Analyzer Release Tracking -Roslyn requires analyzer release tracking documentation. Fixed by adding `$(NoWarn);RS2008` to the generator project. - -### 3. CA1068 - CancellationToken Parameter Order -Method parameters were in wrong order. Fixed by reordering `CancellationToken` to be last. - -### 4. RCS1146 - Conditional Access -Used null check with `&&` instead of `?.` operator. Fixed by using conditional access. - -### 5. CA1310 - StringComparison -`StartsWith(string)` calls without `StringComparison`. Fixed by adding `StringComparison.Ordinal`. - -### 6. CS0103 - Missing Using Directive -Missing `using System;` in SemanticAnalyzer.cs. Fixed by adding the using directive. - -### 7. CS0618 - Obsolete Warnings as Errors -Internal uses of obsolete types caused build failures (TreatWarningsAsErrors). Fixed by adding `#pragma warning disable CS0618` to affected internal files and test files. - -### 8. NU1109 - Package Version Conflict -`Microsoft.CodeAnalysis.Analyzers` 3.3.4 conflicts with `Microsoft.CodeAnalysis.CSharp` 4.14.0 which requires >= 3.11.0. Fixed by updating version to 3.11.0 in `Directory.Packages.props`. - -### 9. RS1041 - Wrong Target Framework for Analyzer -The generator was being multi-targeted due to inherited `TargetFrameworks` from `Directory.Build.props`. Fixed by clearing `TargetFrameworks` and only setting `TargetFramework` to `netstandard2.0`. - -## Migration Guide - -### Before (Reflection-based) - -```csharp -public class MyExecutor : ReflectingExecutor, IMessageHandler -{ - public MyExecutor() : base("MyExecutor") { } - - public ValueTask HandleAsync(MyMessage message, IWorkflowContext context, CancellationToken ct) - { - // Handler implementation - } -} -``` - -### After (Source Generator) - -```csharp -public partial class MyExecutor : Executor -{ - public MyExecutor() : base("MyExecutor") { } - - [MessageHandler] - private ValueTask HandleAsync(MyMessage message, IWorkflowContext context, CancellationToken ct) - { - // Handler implementation - } -} -``` - -Key migration steps: -1. Change base class from `ReflectingExecutor` to `Executor` -2. Add `partial` modifier to the class -3. Remove `IMessageHandler` interface implementations -4. Add `[MessageHandler]` attribute to handler methods -5. Handler methods can now be any accessibility (private, protected, internal, public) - -## Future Work - -- Create comprehensive unit tests for the source generator -- Add integration tests verifying generated routes match reflection-discovered routes -- Consider adding IDE quick-fix for migrating from `ReflectingExecutor` pattern diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index 609c59c078..bd9ff53f18 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -7,6 +7,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.0.0rc6] - 2026-03-30 + +### Added + +- **agent-framework-openai**: New package extracted from core for OpenAI and Azure OpenAI provider support ([#4818](https://github.com/microsoft/agent-framework/pull/4818)) +- **agent-framework-foundry**: New package for Azure AI Foundry integration ([#4818](https://github.com/microsoft/agent-framework/pull/4818)) +- **agent-framework-core**: Support `structuredContent` in MCP tool results and fix sampling options type ([#4763](https://github.com/microsoft/agent-framework/pull/4763)) +- **agent-framework-core**: Include reasoning messages in `MESSAGES_SNAPSHOT` events ([#4844](https://github.com/microsoft/agent-framework/pull/4844)) +- **agent-framework-core**: [BREAKING] Add context mode to `AgentExecutor` ([#4668](https://github.com/microsoft/agent-framework/pull/4668)) + +### Changed + +- **agent-framework-core**: [BREAKING] Remove deprecated kwargs compatibility paths ([#4858](https://github.com/microsoft/agent-framework/pull/4858)) +- **agent-framework-core**: [BREAKING] Reduce core dependencies and simplify optional integrations ([#4904](https://github.com/microsoft/agent-framework/pull/4904)) +- **agent-framework-openai**: [BREAKING] Provider-leading client design & OpenAI package extraction ([#4818](https://github.com/microsoft/agent-framework/pull/4818)) +- **agent-framework-openai**: [BREAKING] Fix OpenAI Azure routing and provider samples ([#4925](https://github.com/microsoft/agent-framework/pull/4925)) +- **agent-framework-azure-ai**: Deprecate Azure AI v1 (Persistent Agents API) helper methods ([#4804](https://github.com/microsoft/agent-framework/pull/4804)) +- **agent-framework-core**: Avoid duplicate agent response telemetry ([#4685](https://github.com/microsoft/agent-framework/pull/4685)) +- **agent-framework-devui**: Bump `flatted` from 3.3.3 to 3.4.2 in frontend ([#4805](https://github.com/microsoft/agent-framework/pull/4805)) +- **samples**: Move `ag_ui_workflow_handoff` demo from `demos/` to `05-end-to-end/` ([#4900](https://github.com/microsoft/agent-framework/pull/4900)) + +### Fixed + +- **agent-framework-core**: Fix streaming path to emit `mcp_server_tool_result` on `output_item.done` instead of `output_item.added` ([#4821](https://github.com/microsoft/agent-framework/pull/4821)) +- **agent-framework-a2a**: Fix `A2AAgent` to surface message content from in-progress `TaskStatusUpdateEvents` ([#4798](https://github.com/microsoft/agent-framework/pull/4798)) +- **agent-framework-core**: Fix `PydanticSchemaGenerationError` when using `from __future__ import annotations` with `@tool` ([#4822](https://github.com/microsoft/agent-framework/pull/4822)) +- **samples**: Fix broken samples for GitHub Copilot, declarative, and Responses API ([#4915](https://github.com/microsoft/agent-framework/pull/4915)) +- **repo**: Fix: update PyRIT repository link from Azure/PyRIT to microsoft/PyRIT ([#4960](https://github.com/microsoft/agent-framework/pull/4960)) + ## [1.0.0rc5] - 2026-03-19 ### Added @@ -817,7 +846,8 @@ Release candidate for **agent-framework-core** and **agent-framework-azure-ai** For more information, see the [announcement blog post](https://devblogs.microsoft.com/foundry/introducing-microsoft-agent-framework-the-open-source-engine-for-agentic-ai-apps/). -[Unreleased]: https://github.com/microsoft/agent-framework/compare/python-1.0.0rc5...HEAD +[Unreleased]: https://github.com/microsoft/agent-framework/compare/python-1.0.0rc6...HEAD +[1.0.0rc6]: https://github.com/microsoft/agent-framework/compare/python-1.0.0rc5...python-1.0.0rc6 [1.0.0rc5]: https://github.com/microsoft/agent-framework/compare/python-1.0.0rc4...python-1.0.0rc5 [1.0.0rc4]: https://github.com/microsoft/agent-framework/compare/python-1.0.0rc3...python-1.0.0rc4 [1.0.0rc3]: https://github.com/microsoft/agent-framework/compare/python-1.0.0rc2...python-1.0.0rc3 diff --git a/python/packages/a2a/pyproject.toml b/python/packages/a2a/pyproject.toml index 52c0d762d1..23b14cd01d 100644 --- a/python/packages/a2a/pyproject.toml +++ b/python/packages/a2a/pyproject.toml @@ -4,7 +4,7 @@ description = "A2A integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "a2a-sdk>=0.3.5,<0.3.24", ] diff --git a/python/packages/ag-ui/pyproject.toml b/python/packages/ag-ui/pyproject.toml index bb1e963023..dbe798fc81 100644 --- a/python/packages/ag-ui/pyproject.toml +++ b/python/packages/ag-ui/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "agent-framework-ag-ui" -version = "1.0.0b260319" +version = "1.0.0b260330" description = "AG-UI protocol integration for Agent Framework" readme = "README.md" license-files = ["LICENSE"] @@ -22,7 +22,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "ag-ui-protocol==0.1.13", "fastapi>=0.115.0,<0.133.1", "uvicorn[standard]>=0.30.0,<0.42.0" diff --git a/python/packages/anthropic/pyproject.toml b/python/packages/anthropic/pyproject.toml index 9b294c0f67..ff89124577 100644 --- a/python/packages/anthropic/pyproject.toml +++ b/python/packages/anthropic/pyproject.toml @@ -4,7 +4,7 @@ description = "Anthropic integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "anthropic>=0.80.0,<0.80.1", ] diff --git a/python/packages/azure-ai-search/pyproject.toml b/python/packages/azure-ai-search/pyproject.toml index af62c00deb..fc5f1e3b35 100644 --- a/python/packages/azure-ai-search/pyproject.toml +++ b/python/packages/azure-ai-search/pyproject.toml @@ -4,7 +4,7 @@ description = "Azure AI Search integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "azure-search-documents>=11.7.0b2,<11.7.0b3", ] diff --git a/python/packages/azure-ai/pyproject.toml b/python/packages/azure-ai/pyproject.toml index 5f0f3a31e1..0a4e0cb66e 100644 --- a/python/packages/azure-ai/pyproject.toml +++ b/python/packages/azure-ai/pyproject.toml @@ -4,7 +4,7 @@ description = "Azure AI Foundry integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0rc5" +version = "1.0.0rc6" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,8 +23,8 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", - "agent-framework-openai>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", + "agent-framework-openai>=1.0.0rc6", "azure-ai-projects>=2.0.0,<3.0", "azure-ai-agents>=1.2.0b5,<1.2.0b6", "azure-ai-inference>=1.0.0b9,<1.0.0b10", diff --git a/python/packages/azure-cosmos/pyproject.toml b/python/packages/azure-cosmos/pyproject.toml index cbb8188de0..d1bc91d3c1 100644 --- a/python/packages/azure-cosmos/pyproject.toml +++ b/python/packages/azure-cosmos/pyproject.toml @@ -4,7 +4,7 @@ description = "Azure Cosmos DB history provider integration for Microsoft Agent authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "azure-cosmos>=4.3.0,<5", ] diff --git a/python/packages/azurefunctions/pyproject.toml b/python/packages/azurefunctions/pyproject.toml index be8dee5e4a..83f48ea169 100644 --- a/python/packages/azurefunctions/pyproject.toml +++ b/python/packages/azurefunctions/pyproject.toml @@ -4,7 +4,7 @@ description = "Azure Functions integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -22,7 +22,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "agent-framework-durabletask", "azure-functions>=1.24.0,<2", "azure-functions-durable>=1.3.1,<2", diff --git a/python/packages/bedrock/pyproject.toml b/python/packages/bedrock/pyproject.toml index 90b134e068..1583d44098 100644 --- a/python/packages/bedrock/pyproject.toml +++ b/python/packages/bedrock/pyproject.toml @@ -4,7 +4,7 @@ description = "Amazon Bedrock integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "boto3>=1.35.0,<2.0.0", "botocore>=1.35.0,<2.0.0", ] diff --git a/python/packages/chatkit/pyproject.toml b/python/packages/chatkit/pyproject.toml index 679891d115..7da827adcb 100644 --- a/python/packages/chatkit/pyproject.toml +++ b/python/packages/chatkit/pyproject.toml @@ -4,7 +4,7 @@ description = "OpenAI ChatKit integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -22,7 +22,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "openai-chatkit>=1.4.1,<2.0.0", ] diff --git a/python/packages/claude/pyproject.toml b/python/packages/claude/pyproject.toml index 696f06cb9f..b78e00cf42 100644 --- a/python/packages/claude/pyproject.toml +++ b/python/packages/claude/pyproject.toml @@ -4,7 +4,7 @@ description = "Claude Agent SDK integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "claude-agent-sdk>=0.1.36,<0.1.49", ] diff --git a/python/packages/copilotstudio/pyproject.toml b/python/packages/copilotstudio/pyproject.toml index 846325b2ce..7a29537b7e 100644 --- a/python/packages/copilotstudio/pyproject.toml +++ b/python/packages/copilotstudio/pyproject.toml @@ -4,7 +4,7 @@ description = "Copilot Studio integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "microsoft-agents-copilotstudio-client>=0.3.1,<0.3.2", ] diff --git a/python/packages/core/agent_framework/__init__.py b/python/packages/core/agent_framework/__init__.py index 0f652f23bd..a9e4245e77 100644 --- a/python/packages/core/agent_framework/__init__.py +++ b/python/packages/core/agent_framework/__init__.py @@ -57,6 +57,27 @@ from ._compaction import ( included_messages, included_token_count, ) +from ._evaluation import ( + AgentEvalConverter, + CheckResult, + ConversationSplit, + ConversationSplitter, + EvalItem, + EvalItemResult, + EvalNotPassedError, + EvalResults, + EvalScoreResult, + Evaluator, + ExpectedToolCall, + LocalEvaluator, + evaluate_agent, + evaluate_workflow, + evaluator, + keyword_check, + tool_call_args_match, + tool_called_check, + tool_calls_present, +) from ._mcp import MCPStdioTool, MCPStreamableHTTPTool, MCPWebsocketTool from ._middleware import ( AgentContext, @@ -242,6 +263,7 @@ __all__ = [ "USER_AGENT_TELEMETRY_DISABLED_ENV_VAR", "Agent", "AgentContext", + "AgentEvalConverter", "AgentExecutor", "AgentExecutorRequest", "AgentExecutorResponse", @@ -268,11 +290,14 @@ __all__ = [ "ChatOptions", "ChatResponse", "ChatResponseUpdate", + "CheckResult", "CheckpointStorage", "CompactionProvider", "CompactionStrategy", "Content", "ContinuationToken", + "ConversationSplit", + "ConversationSplitter", "Default", "Edge", "EdgeCondition", @@ -281,7 +306,14 @@ __all__ = [ "EmbeddingGenerationOptions", "EmbeddingInputT", "EmbeddingT", + "EvalItem", + "EvalItemResult", + "EvalNotPassedError", + "EvalResults", + "EvalScoreResult", + "Evaluator", "Executor", + "ExpectedToolCall", "FanInEdgeGroup", "FanOutEdgeGroup", "FileCheckpointStorage", @@ -300,6 +332,7 @@ __all__ = [ "InMemoryCheckpointStorage", "InMemoryHistoryProvider", "InProcRunnerContext", + "LocalEvaluator", "MCPStdioTool", "MCPStreamableHTTPTool", "MCPWebsocketTool", @@ -379,11 +412,15 @@ __all__ = [ "chat_middleware", "create_edge_runner", "detect_media_type_from_base64", + "evaluate_agent", + "evaluate_workflow", + "evaluator", "executor", "function_middleware", "handler", "included_messages", "included_token_count", + "keyword_check", "load_settings", "map_chat_to_agent_update", "merge_chat_options", @@ -396,6 +433,9 @@ __all__ = [ "resolve_agent_id", "response_handler", "tool", + "tool_call_args_match", + "tool_called_check", + "tool_calls_present", "validate_chat_options", "validate_tool_mode", "validate_tools", diff --git a/python/packages/core/agent_framework/_evaluation.py b/python/packages/core/agent_framework/_evaluation.py new file mode 100644 index 0000000000..92a694cc36 --- /dev/null +++ b/python/packages/core/agent_framework/_evaluation.py @@ -0,0 +1,1906 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Provider-agnostic evaluation framework for Microsoft Agent Framework. + +Defines the core evaluation types and orchestration functions that work with +any evaluation provider (Azure AI Foundry, local evaluators, third-party +libraries, etc.). Also includes ``LocalEvaluator`` and built-in check +functions for fast, API-free evaluation during inner-loop development and +CI smoke tests. + +Cloud evaluator example: + +.. code-block:: python + + from agent_framework import evaluate_agent, EvalResults + from agent_framework.foundry import FoundryEvals + + evals = FoundryEvals(project_client=client, model="gpt-4o") + results = await evaluate_agent(agent=agent, queries=["Hello"], evaluators=evals) + results.raise_for_status() + +Local evaluator example: + +.. code-block:: python + + from agent_framework import LocalEvaluator, keyword_check, evaluate_agent + + local = LocalEvaluator( + keyword_check("weather", "temperature"), + tool_called_check("get_weather"), + ) + results = await evaluate_agent(agent=agent, queries=queries, evaluators=local) +""" + +from __future__ import annotations + +import asyncio +import contextlib +import inspect +import json +import logging +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass, field +from enum import Enum +from typing import ( + TYPE_CHECKING, + Any, + Literal, + Protocol, + TypedDict, + cast, + overload, + runtime_checkable, +) + +from ._tools import FunctionTool +from ._types import AgentResponse, Message + +if TYPE_CHECKING: + from ._agents import SupportsAgentRun + from ._workflows._agent_executor import AgentExecutorResponse + from ._workflows._workflow import Workflow, WorkflowRunResult + +logger = logging.getLogger(__name__) + + +class EvalNotPassedError(Exception): + """Raised when evaluation results contain failures.""" + + +# region Core types + + +@runtime_checkable +class ConversationSplitter(Protocol): + """Strategy for splitting a conversation into (query, response) messages. + + Any callable with this signature satisfies the protocol — including the + built-in ``ConversationSplit`` enum members and custom functions: + + .. code-block:: python + + def my_splitter(conversation: list[Message]) -> tuple[list[Message], list[Message]]: + '''Return (query_messages, response_messages).''' + + Custom splitters let you evaluate domain-specific boundaries — for example, + splitting just before a memory-retrieval tool call to evaluate recall quality: + + .. code-block:: python + + def split_before_memory(conversation): + for i, msg in enumerate(conversation): + for c in msg.contents or []: + if c.type == "function_call" and c.name == "retrieve_memory": + return conversation[:i], conversation[i:] + # Fallback: split at last user message + return EvalItem._split_last_turn_static(conversation) + + item.split_messages(split=split_before_memory) + """ + + def __call__(self, conversation: list[Message]) -> tuple[list[Message], list[Message]]: ... + + +class ConversationSplit(str, Enum): + """Built-in conversation split strategies. + + Each member is callable, satisfying the ``ConversationSplitter`` protocol:: + + query_msgs, response_msgs = ConversationSplit.LAST_TURN(conversation) + + - ``LAST_TURN``: Split at the last user message. Everything up to and + including that message is the query; everything after is the response. + Evaluates whether the agent answered the *latest* question well. + + - ``FULL``: The first user message (and any preceding system messages) is + the query; the entire remainder of the conversation is the response. + Evaluates whether the *whole conversation trajectory* served the + original request. + + For custom splits, pass any callable with the ``ConversationSplitter`` + signature. + """ + + LAST_TURN = "last_turn" + FULL = "full" + + def __call__(self, conversation: list[Message]) -> tuple[list[Message], list[Message]]: + """Dispatch to the built-in splitter implementation.""" + return _BUILT_IN_SPLITTERS[self](conversation) + + +@dataclass +class ExpectedToolCall: + """A tool call that an agent is expected to make. + + Used with :func:`evaluate_agent` to assert that the agent called the + correct tools. The *evaluator* decides the matching semantics (order, + extras, argument checking); this type is pure data. + + Attributes: + name: The tool/function name (e.g. ``"get_weather"``). + arguments: Expected arguments. ``None`` means "don't check arguments" or "no arguments". + """ + + name: str + arguments: dict[str, Any] | None = None + + +def _split_last_turn(conversation: list[Message]) -> tuple[list[Message], list[Message]]: + """Split at the last user message (default strategy).""" + last_user_idx = -1 + for i, msg in enumerate(conversation): + if msg.role == "user": + last_user_idx = i + if last_user_idx >= 0: + return conversation[: last_user_idx + 1], conversation[last_user_idx + 1 :] + return [], list(conversation) + + +def _split_full(conversation: list[Message]) -> tuple[list[Message], list[Message]]: + """Split after the first user message (evaluates whole trajectory).""" + for i, msg in enumerate(conversation): + if msg.role == "user": + return conversation[: i + 1], conversation[i + 1 :] + return [], list(conversation) + + +_BUILT_IN_SPLITTERS: dict[ConversationSplit, Callable[[list[Message]], tuple[list[Message], list[Message]]]] = { + ConversationSplit.LAST_TURN: _split_last_turn, + ConversationSplit.FULL: _split_full, +} + + +class EvalItem: + """A single item to be evaluated. + + Represents one query/response interaction in a provider-agnostic format. + ``conversation`` is the single source of truth — ``query`` and ``response`` + are derived from it via the split strategy. + + Attributes: + conversation: Full conversation as ``Message`` objects. + tools: Typed tool objects (e.g. ``FunctionTool``) for evaluator logic. + context: Optional grounding context document. + expected_output: Optional expected output for ground-truth comparison. + expected_tool_calls: Expected tool calls for tool-correctness + evaluation. See :class:`ExpectedToolCall`. + split_strategy: Split strategy controlling how ``query`` and + ``response`` are derived from the conversation. Defaults to + ``ConversationSplit.LAST_TURN``. + """ + + def __init__( + self, + conversation: list[Message], + tools: list[FunctionTool] | None = None, + context: str | None = None, + expected_output: str | None = None, + expected_tool_calls: list[ExpectedToolCall] | None = None, + split_strategy: ConversationSplitter | None = None, + ) -> None: + self.conversation = conversation + self.tools = tools + self.context = context + self.expected_output = expected_output + self.expected_tool_calls = expected_tool_calls + self.split_strategy = split_strategy + + @property + def query(self) -> str: + """User query text, derived from the query side of the conversation split.""" + query_msgs, _ = self._split_conversation(self.split_strategy or ConversationSplit.LAST_TURN) + user_texts = [m.text for m in query_msgs if m.role == "user" and m.text] + return " ".join(user_texts).strip() + + @property + def response(self) -> str: + """Agent response text, derived from the response side of the conversation split.""" + _, response_msgs = self._split_conversation(self.split_strategy or ConversationSplit.LAST_TURN) + assistant_texts = [m.text for m in response_msgs if m.role == "assistant" and m.text] + return " ".join(assistant_texts).strip() + + def _split_conversation(self, split: ConversationSplitter) -> tuple[list[Message], list[Message]]: + """Split ``self.conversation`` into (query_messages, response_messages).""" + return split(self.conversation) + + def split_messages( + self, + split: ConversationSplitter | None = None, + ) -> tuple[list[Message], list[Message]]: + """Split the conversation into (query_messages, response_messages). + + Resolution order: explicit *split*, then ``self.split_strategy``, + then ``ConversationSplit.LAST_TURN``. + """ + effective = split or self.split_strategy or ConversationSplit.LAST_TURN + return self._split_conversation(effective) + + @staticmethod + def _split_last_turn_static( + conversation: list[Message], + ) -> tuple[list[Message], list[Message]]: + """Split at the last user message. Usable as a fallback in custom splitters.""" + return _split_last_turn(conversation) + + @staticmethod + def per_turn_items( + conversation: list[Message], + *, + tools: list[FunctionTool] | None = None, + context: str | None = None, + ) -> list[EvalItem]: + """Split a multi-turn conversation into one ``EvalItem`` per turn. + + Each user message starts a new turn. The resulting ``EvalItem`` + has cumulative context: ``query_messages`` contains the full + conversation up to and including that user message, and + ``response_messages`` contains the agent's actions up to the next + user message. This lets you evaluate each response independently + with its full preceding context. + + Args: + conversation: Full conversation as ``Message`` objects. + tools: Tool objects shared across all items. + context: Optional grounding context shared across all items. + + Returns: + A list of ``EvalItem`` instances, one per user turn. + """ + user_indices = [i for i, m in enumerate(conversation) if m.role == "user"] + if not user_indices: + return [] + + items: list[EvalItem] = [] + for turn_idx, _ui in enumerate(user_indices): + # Response runs from after the user message to the next user + # message (or end of conversation). + next_ui = user_indices[turn_idx + 1] if turn_idx + 1 < len(user_indices) else len(conversation) + + items.append( + EvalItem( + conversation=conversation[:next_ui], + tools=tools, + context=context, + ) + ) + + return items + + +# endregion + +# region Score and result types + + +@dataclass +class EvalScoreResult: + """Result from a single evaluator on a single item. + + Attributes: + name: Evaluator name (e.g. ``"relevance"``). + score: Numeric score from the evaluator. + passed: Whether the item passed this evaluator's threshold. + sample: Optional raw evaluator output (rationale, metadata). + """ + + name: str + score: float + passed: bool | None = None + sample: dict[str, Any] | None = None + + +@dataclass +class EvalItemResult: + """Per-item result from an evaluation run. + + Attributes: + item_id: Provider-assigned item identifier. + status: ``"pass"``, ``"fail"``, or ``"error"``. + scores: Per-evaluator results for this item. + error_code: Error category when ``status == "error"`` + (e.g. ``"QueryExtractionError"``). + error_message: Human-readable error detail. + response_id: Responses API response ID, if applicable. + input_text: The query/input that was evaluated. + output_text: The response/output that was evaluated. + token_usage: Token counts (``prompt_tokens``, + ``completion_tokens``, ``total_tokens``). + metadata: Additional provider-specific data. + """ + + item_id: str + status: str + scores: list[EvalScoreResult] = field(default_factory=lambda: list[EvalScoreResult]()) + error_code: str | None = None + error_message: str | None = None + response_id: str | None = None + input_text: str | None = None + output_text: str | None = None + token_usage: dict[str, int] | None = None + metadata: dict[str, Any] | None = None + + @property + def is_error(self) -> bool: + """Whether this item errored (infrastructure failure, not quality).""" + return self.status in ("error", "errored") + + @property + def is_passed(self) -> bool: + """Whether this item passed all evaluators.""" + return self.status == "pass" + + @property + def is_failed(self) -> bool: + """Whether this item failed at least one evaluator.""" + return self.status == "fail" + + +class EvalResults: + """Results from an evaluation run by a single provider. + + Attributes: + provider: Name of the evaluation provider that produced these results. + eval_id: The evaluation definition ID (provider-specific). + run_id: The evaluation run ID (provider-specific). + status: Run status - ``"completed"``, ``"failed"``, ``"canceled"``, + or ``"timeout"`` if polling exceeded the deadline. + result_counts: Pass/fail counts, populated when completed. + report_url: URL to view results in the provider's portal. + error: Error details when the run failed. + per_evaluator: Per-evaluator result counts, keyed by evaluator name. + items: Per-item results with individual pass/fail/error status, + evaluator scores, error details, and token usage. Populated + when the provider supports per-item retrieval (e.g. Foundry + ``output_items`` API). + sub_results: Per-agent breakdown for workflow evaluations, keyed by + agent/executor name. + + Example: + + .. code-block:: python + + results = await evaluate_agent(agent=my_agent, queries=["Hello"], evaluators=evals) + for r in results: + print(f"{r.provider}: {r.passed}/{r.total}") + + # Per-item detail + for item in r.items: + print(f" {item.item_id}: {item.status}") + for score in item.scores: + print(f" {score.name}: {score.score} ({'pass' if score.passed else 'fail'})") + if item.is_error: + print(f" Error: {item.error_code} - {item.error_message}") + + # Workflow eval - per-agent breakdown + for r in results: + for name, sub in r.sub_results.items(): + print(f" {name}: {sub.passed}/{sub.total}") + """ + + def __init__( + self, + *, + provider: str, + eval_id: str = "", + run_id: str = "", + status: str = "completed", + result_counts: dict[str, int] | None = None, + report_url: str | None = None, + error: str | None = None, + per_evaluator: dict[str, dict[str, int]] | None = None, + items: list[EvalItemResult] | None = None, + sub_results: dict[str, EvalResults] | None = None, + ) -> None: + self.provider = provider + self.eval_id = eval_id + self.run_id = run_id + self.status = status + self.result_counts = result_counts + self.report_url = report_url + self.error = error + self.per_evaluator = per_evaluator or {} + self.items = items or [] + self.sub_results = sub_results or {} + + @property + def passed(self) -> int: + """Number of passing results.""" + return (self.result_counts or {}).get("passed", 0) + + @property + def failed(self) -> int: + """Number of failing results.""" + return (self.result_counts or {}).get("failed", 0) + + @property + def total(self) -> int: + """Total number of results (passed + failed).""" + return self.passed + self.failed + + @property + def all_passed(self) -> bool: + """Whether all results passed with no failures or errors. + + For workflow evals with sub-agents, checks that all sub-results passed. + Returns ``False`` if the run did not complete successfully. + """ + if self.status not in ("completed",): + return False + errored = (self.result_counts or {}).get("errored", 0) + own_passed = self.failed == 0 and errored == 0 and self.total > 0 if self.result_counts else True + if self.sub_results: + return own_passed and all(sub.all_passed for sub in self.sub_results.values()) + return self.failed == 0 and errored == 0 and self.total > 0 + + def raise_for_status(self, msg: str | None = None) -> None: + """Raise ``EvalNotPassedError`` if any results failed or errored. + + Similar to ``requests.Response.raise_for_status()`` — call after + evaluation to verify quality in CI pipelines or test suites. + + Args: + msg: Optional custom failure message. + + Raises: + EvalNotPassedError: When any results failed or errored. + """ + if not self.all_passed: + errored = (self.result_counts or {}).get("errored", 0) + detail = msg or ( + f"Eval run {self.run_id} {self.status}: " + f"{self.passed} passed, {self.failed} failed." + ) + if errored: + detail += f" {errored} errored." + if self.report_url: + detail += f" See {self.report_url} for details." + if self.error: + detail += f" Error: {self.error}" + if self.sub_results: + failed = [name for name, sub in self.sub_results.items() if not sub.all_passed] + if failed: + detail += f" Failed: {', '.join(failed)}." + if self.items: + errored_items = [i for i in self.items if i.is_error] + if errored_items: + summaries = [f"{i.item_id}: {i.error_code or 'unknown'}" for i in errored_items] + detail += f" Errored items: {', '.join(summaries)}." + raise EvalNotPassedError(detail) + + +# endregion + +# region Evaluator protocol + + +@runtime_checkable +class Evaluator(Protocol): + """Protocol for evaluation providers. + + Any evaluation backend (Azure AI Foundry, local LLM-as-judge, custom + scorers, etc.) implements this protocol. The provider encapsulates all + connection details, evaluator selection, and execution logic. + + Example implementation: + + .. code-block:: python + + class MyEvaluator: + def __init__(self, name: str = "my-evaluator"): + self.name = name + + async def evaluate(self, items: Sequence[EvalItem], *, eval_name: str = "Eval") -> EvalResults: + # Score each item and return results + ... + """ + + name: str + + async def evaluate( + self, + items: Sequence[EvalItem], + *, + eval_name: str, + ) -> EvalResults: + """Evaluate a batch of items and return results. + + The evaluator determines which metrics to run. It may auto-detect + capabilities from the items (e.g., run tool evaluators only when + ``tools`` is present). + + Args: + items: Eval data items to score. + eval_name: Display name for the evaluation run. + + Returns: + ``EvalResults`` with status, counts, and optional portal link. + """ + ... + + +# endregion + +# region Converter + + +class AgentEvalConverter: + """Converts agent-framework types to evaluation format. + + Handles the type gap between agent-framework's ``Message`` / ``Content`` / + ``FunctionTool`` types and the OpenAI-style agent message schema used by + evaluation providers. All methods are static — no instantiation needed. + """ + + @staticmethod + def convert_message(message: Message) -> list[dict[str, Any]]: + """Convert a single ``Message`` to Foundry agent evaluator format. + + Uses typed content lists as required by Foundry evaluators: + + .. code-block:: python + + {"role": "assistant", "content": [{"type": "tool_call", ...}]} + {"role": "user", "content": [{"type": "input_image", ...}]} + + Supported content types: + + * ``text`` → ``{"type": "text", "text": ...}`` + * ``data`` / ``uri`` (images) → ``{"type": "input_image", "image_url": ...}`` + * ``function_call`` → ``{"type": "tool_call", ...}`` + * ``function_result`` → ``{"type": "tool_result", ...}`` + + A single agent-framework ``Message`` with multiple ``function_result`` + contents produces multiple output messages (one per tool result). + + Args: + message: An agent-framework ``Message``. + + Returns: + A list of Foundry-format message dicts. + """ + role = message.role + contents = message.contents or [] + + content_items: list[dict[str, Any]] = [] + tool_results: list[dict[str, Any]] = [] + + for c in contents: + if c.type == "text" and c.text: + content_items.append({"type": "text", "text": c.text}) + elif c.type in ("data", "uri") and c.uri: + # Image / media content → OpenAI input_image format + img: dict[str, Any] = { + "type": "input_image", + "image_url": c.uri, + } + if c.media_type: + img["detail"] = "auto" + content_items.append(img) + elif c.type == "function_call": + args = c.arguments + if isinstance(args, str): + try: + args = json.loads(args) + except (json.JSONDecodeError, TypeError): + # Sanitize to avoid leaking sensitive tool-call arguments + # to external evaluation services. + args = {"_raw_arguments": "[unparseable]"} + tc: dict[str, Any] = { + "type": "tool_call", + "tool_call_id": c.call_id or "", + "name": c.name or "", + } + if args: + tc["arguments"] = args + content_items.append(tc) + elif c.type == "function_result": + result_val = c.result + if isinstance(result_val, str): + with contextlib.suppress(json.JSONDecodeError, TypeError): + result_val = json.loads(result_val) + tool_results.append({ + "call_id": c.call_id or "", + "result": result_val, + }) + + output: list[dict[str, Any]] = [] + + if tool_results: + for tr in tool_results: + output.append({ + "role": "tool", + "tool_call_id": tr["call_id"], + "content": [{"type": "tool_result", "tool_result": tr["result"]}], + }) + elif content_items: + output.append({"role": role, "content": content_items}) + else: + output.append({ + "role": role, + "content": [{"type": "text", "text": ""}], + }) + + return output + + @staticmethod + def convert_messages(messages: Sequence[Message]) -> list[dict[str, Any]]: + """Convert a sequence of ``Message`` objects to Foundry evaluator format. + + Args: + messages: Agent-framework messages. + + Returns: + A list of Foundry-format message dicts with typed content lists. + """ + result: list[dict[str, Any]] = [] + for msg in messages: + result.extend(AgentEvalConverter.convert_message(msg)) + return result + + @staticmethod + def extract_tools(agent: Any) -> list[dict[str, Any]]: + """Extract tool definitions from an agent instance. + + Reads ``agent.default_options["tools"]`` and ``agent.mcp_tools`` + and converts each ``FunctionTool`` to ``{name, description, parameters}``. + + Args: + agent: An agent-framework agent instance. + + Returns: + A list of tool definition dicts. + """ + tools: list[dict[str, Any]] = [] + seen: set[str] = set() + raw_tools = getattr(agent, "default_options", {}).get("tools", []) + for t in raw_tools: + if isinstance(t, FunctionTool) and t.name not in seen: + tools.append({ + "name": t.name, + "description": t.description, + "parameters": t.parameters(), + }) + seen.add(t.name) + # Include tools from connected MCP servers + for mcp in getattr(agent, "mcp_tools", []): + for t in getattr(mcp, "functions", []): + if isinstance(t, FunctionTool) and t.name not in seen: + tools.append({ + "name": t.name, + "description": t.description, + "parameters": t.parameters(), + }) + seen.add(t.name) + return tools + + @staticmethod + def to_eval_item( + *, + query: str | Sequence[Message], + response: AgentResponse[Any], + agent: Any | None = None, + tools: Sequence[FunctionTool] | None = None, + context: str | None = None, + ) -> EvalItem: + """Convert a complete agent interaction to an ``EvalItem``. + + Args: + query: The user query string, or input messages. + response: The agent's response. + agent: Optional agent instance to auto-extract tool definitions. + tools: Explicit tool list (takes precedence over *agent*). + context: Optional context document for groundedness evaluation. + + Returns: + An ``EvalItem`` suitable for passing to any ``Evaluator``. + """ + input_msgs = [Message("user", [query])] if isinstance(query, str) else list(query) + + all_msgs = list(input_msgs) + list(response.messages or []) + + typed_tools: list[FunctionTool] = [] + if tools: + typed_tools = list(tools) + elif agent: + raw_tools = getattr(agent, "default_options", {}).get("tools", []) + typed_tools = [t for t in raw_tools if isinstance(t, FunctionTool)] + # Include tools from connected MCP servers + seen = {t.name for t in typed_tools} + for mcp in getattr(agent, "mcp_tools", []): + for t in getattr(mcp, "functions", []): + if isinstance(t, FunctionTool) and t.name not in seen: + typed_tools.append(t) + seen.add(t.name) + + return EvalItem( + conversation=all_msgs, + tools=typed_tools or None, + context=context, + ) + + +# endregion + +# region Workflow extraction helpers + + +class _AgentEvalData(TypedDict): + executor_id: str + query: str | Sequence[Message] + response: AgentResponse[Any] + agent: Any | None + + +def _extract_agent_eval_data( + workflow_result: WorkflowRunResult, + workflow: Workflow | None = None, +) -> list[_AgentEvalData]: + """Walk a WorkflowRunResult and extract per-agent query/response pairs. + + Pairs ``executor_invoked`` with ``executor_completed`` events for each + ``AgentExecutor``. Skips internal framework executors. + """ + from ._workflows._agent_executor import AgentExecutor as AE + from ._workflows._agent_executor import AgentExecutorResponse + + invoked_data: dict[str, Any] = {} + results: list[_AgentEvalData] = [] + + for event in workflow_result: + if event.type == "executor_invoked" and event.executor_id: + invoked_data[event.executor_id] = event.data + + elif event.type == "executor_completed" and event.executor_id: + executor_id = event.executor_id + + # Skip internal framework executors + if executor_id.startswith("_") or executor_id.lower() in {"input-conversation", "end-conversation", "end"}: + logger.debug("Skipping internal executor %r during eval data extraction", executor_id) + continue + + completion_data: Any = event.data + agent_exec_response: AgentExecutorResponse | None = None + + if isinstance(completion_data, list): + for cdata_item in cast(list[Any], completion_data): # type: ignore[redundant-cast] + if isinstance(cdata_item, AgentExecutorResponse): + agent_exec_response = cdata_item + break + elif isinstance(completion_data, AgentExecutorResponse): + agent_exec_response = completion_data + + if agent_exec_response is None: + continue + + query: str | list[Message] + if agent_exec_response.full_conversation: + user_msgs = [m for m in agent_exec_response.full_conversation if m.role == "user"] + query = user_msgs or agent_exec_response.full_conversation # type: ignore[assignment] + elif executor_id in invoked_data: + input_data: Any = invoked_data[executor_id] + query = ( # type: ignore[assignment] + input_data if isinstance(input_data, (str, list)) else str(input_data) + ) + else: + continue + + agent_ref = None + if workflow is not None: + executor = workflow.executors.get(executor_id) + if executor is not None and isinstance(executor, AE): + agent_ref = executor.agent + + results.append( + _AgentEvalData( + executor_id=executor_id, + query=query, + response=agent_exec_response.agent_response, + agent=agent_ref, + ) + ) + + return results + + +def _extract_overall_query(workflow_result: WorkflowRunResult) -> str | None: + """Extract the original user query from a workflow result.""" + for event in workflow_result: + if event.type == "executor_invoked" and event.data is not None: + data: Any = event.data + if isinstance(data, str): + return data + if isinstance(data, list) and data: + items_list = cast(list[Any], data) # type: ignore[redundant-cast] + first = items_list[0] + if isinstance(first, Message): + msgs: list[Message] = [m for m in items_list if isinstance(m, Message)] + return " ".join(str(m.text) for m in msgs if hasattr(m, "text") and m.role == "user") + if isinstance(first, str): + return " ".join(str(s) for s in items_list) + return str(data) # type: ignore[reportUnknownArgumentType] + return None + + +# endregion + +# region Local evaluation checks + + +@dataclass +class CheckResult: + """Result of a single check on a single evaluation item. + + Attributes: + passed: Whether the check passed. + reason: Human-readable explanation. + check_name: Name of the check that produced this result. + """ + + passed: bool + reason: str + check_name: str + + +EvalCheck = Callable[[EvalItem], CheckResult | Awaitable[CheckResult]] +"""A check function that takes an ``EvalItem`` and returns a ``CheckResult``. + +Both sync and async functions are supported. Async checks should return +an awaitable ``CheckResult``; they will be awaited automatically by +``LocalEvaluator``. +""" + + +def keyword_check(*keywords: str, case_sensitive: bool = False) -> EvalCheck: + """Check that the response contains all specified keywords. + + Args: + *keywords: Required keywords that must appear in the response. + case_sensitive: Whether matching is case-sensitive (default ``False``). + + Returns: + A check function for use with ``LocalEvaluator``. + + Example: + + .. code-block:: python + + check = keyword_check("weather", "temperature") + """ + + def _check(item: EvalItem) -> CheckResult: + text = item.response if case_sensitive else item.response.lower() + missing = [k for k in keywords if (k if case_sensitive else k.lower()) not in text] + if missing: + return CheckResult(passed=False, reason=f"Missing keywords: {missing}", check_name="keyword_check") + return CheckResult(passed=True, reason="All keywords found", check_name="keyword_check") + + return _check + + +def tool_called_check(*tool_names: str, mode: Literal["all", "any"] = "all") -> EvalCheck: + """Check that specific tools were called during the conversation. + + Inspects the conversation history for ``tool_calls`` entries matching + the expected tool names. + + Args: + *tool_names: Names of tools that should have been called. + mode: ``"all"`` requires every tool to be called; ``"any"`` requires + at least one. Defaults to ``"all"``. + + Returns: + A check function for use with ``LocalEvaluator``. + + Example: + + .. code-block:: python + + check = tool_called_check("get_weather", "get_flight_price") + """ + + def _check(item: EvalItem) -> CheckResult: + expected = set(tool_names) + called: set[str] = set() + for msg in item.conversation: + for c in msg.contents or []: + if c.type == "function_call" and c.name: + called.add(c.name) + if mode == "all" and expected.issubset(called): + return CheckResult( + passed=True, + reason=f"All expected tools called: {sorted(called)}", + check_name="tool_called", + ) + if mode == "any" and expected & called: + return CheckResult( + passed=True, + reason=f"Expected tool found: {sorted(expected & called)}", + check_name="tool_called", + ) + if mode == "all": + missing = [t for t in tool_names if t not in called] + if missing: + return CheckResult( + passed=False, + reason=f"Expected tools not called: {missing} (called: {sorted(called)})", + check_name="tool_called", + ) + return CheckResult( + passed=True, + reason=f"All expected tools called: {sorted(called)}", + check_name="tool_called", + ) + return CheckResult( + passed=False, + reason=f"None of expected tools called: {list(tool_names)} (called: {sorted(called)})", + check_name="tool_called", + ) + + return _check + + +def _extract_tool_calls(item: EvalItem) -> list[tuple[str, dict[str, Any] | None]]: + """Extract (name, arguments) pairs from the conversation's function calls.""" + calls: list[tuple[str, dict[str, Any] | None]] = [] + for msg in item.conversation: + for c in msg.contents or []: + if c.type == "function_call" and c.name: + args: dict[str, Any] | None = None + if isinstance(c.arguments, dict): + args = c.arguments + elif isinstance(c.arguments, str): + try: + parsed = json.loads(c.arguments) + if isinstance(parsed, dict): + args = cast(dict[str, Any], parsed) + except (json.JSONDecodeError, TypeError): + pass + calls.append((c.name, args)) + return calls + + +def tool_calls_present(item: EvalItem) -> CheckResult: + """Check that all expected tool calls were made (unordered, extras OK). + + Uses ``item.expected_tool_calls`` — checks that every expected tool name + appears at least once in the conversation. Does not check arguments or + ordering. Extra (unexpected) tool calls are not penalized. + + Example: + + .. code-block:: python + + local = LocalEvaluator(tool_calls_present) + results = await evaluate_agent( + agent=agent, + queries=["What's the weather?"], + expected_tool_calls=[[ExpectedToolCall("get_weather")]], + evaluators=local, + ) + """ + expected = item.expected_tool_calls or [] + if not expected: + return CheckResult(passed=True, reason="No expected tool calls specified.", check_name="tool_calls_present") + + actual_names = {name for name, _ in _extract_tool_calls(item)} + expected_names = [e.name for e in expected] + found = [n for n in expected_names if n in actual_names] + missing = [n for n in expected_names if n not in actual_names] + + if missing: + return CheckResult( + passed=False, + reason=f"Missing tool calls: {missing} (called: {sorted(actual_names)})", + check_name="tool_calls_present", + ) + return CheckResult( + passed=True, + reason=f"All expected tools called: {found} (called: {sorted(actual_names)})", + check_name="tool_calls_present", + ) + + +def tool_call_args_match(item: EvalItem) -> CheckResult: + """Check that expected tool calls match on name and arguments. + + For each expected tool call, finds matching calls in the conversation + by name. If ``ExpectedToolCall.arguments`` is provided, checks that + the actual arguments contain all expected key-value pairs (subset + match — extra actual arguments are OK). + + Example: + + .. code-block:: python + + local = LocalEvaluator(tool_call_args_match) + results = await evaluate_agent( + agent=agent, + queries=["What's the weather in NYC?"], + expected_tool_calls=[ + [ExpectedToolCall("get_weather", {"location": "NYC"})], + ], + evaluators=local, + ) + """ + expected = item.expected_tool_calls or [] + if not expected: + return CheckResult(passed=True, reason="No expected tool calls specified.", check_name="tool_call_args_match") + + actual_calls = _extract_tool_calls(item) + matched = 0 + details: list[str] = [] + + for exp in expected: + matching = [(n, a) for n, a in actual_calls if n == exp.name] + if not matching: + details.append(f" {exp.name}: not called") + continue + + if exp.arguments is None: + matched += 1 + details.append(f" {exp.name}: called (args not checked)") + continue + + # Subset match — all expected keys present with expected values + found = False + for _, actual_args in matching: + if actual_args is None: + continue + if all(actual_args.get(k) == v for k, v in exp.arguments.items()): + found = True + break + + if found: + matched += 1 + details.append(f" {exp.name}: args match") + else: + actual_args_list = [a for _, a in matching] + details.append(f" {exp.name}: args mismatch (actual: {actual_args_list})") + + passed = matched == len(expected) + score_str = f"{matched}/{len(expected)}" + detail_str = "\n".join(details) + reason = f"Tool call args match: {score_str}\n{detail_str}" + + return CheckResult(passed=passed, reason=reason, check_name="tool_call_args_match") + + +# endregion + +# region Function evaluator — wrap plain functions as EvalChecks + +# Parameters recognized by the function evaluator wrapper +_KNOWN_PARAMS = frozenset({ + "query", + "response", + "expected_output", + "expected_tool_calls", + "conversation", + "tools", + "context", +}) + + +def _resolve_function_args( + fn: Callable[..., Any], + item: EvalItem, + *, + _param_names: frozenset[str] | set[str] | None = None, +) -> dict[str, Any]: + """Build a kwargs dict for *fn* based on its signature and the EvalItem. + + Supported parameter names: + + ====================== ==================================================== + Name Value from EvalItem + ====================== ==================================================== + query ``item.query`` + response ``item.response`` + expected_output ``item.expected_output`` (empty string if not set) + expected_tool_calls ``item.expected_tool_calls`` (empty list if not set) + conversation ``item.conversation`` (list[Message]) + tools ``item.tools`` (typed ``FunctionTool`` objects) + context ``item.context`` + ====================== ==================================================== + + Parameters with default values are only supplied when their name is + recognised. Unknown required parameters raise ``TypeError``. + + When called from the ``@evaluator`` wrapper the pre-computed + *_param_names* set avoids repeated ``inspect.signature`` calls. + """ + field_map: dict[str, Any] = { + "query": item.query, + "response": item.response, + "expected_output": item.expected_output or "", + "expected_tool_calls": item.expected_tool_calls or [], + "conversation": item.conversation, + "tools": item.tools, + "context": item.context, + } + + if _param_names is not None: + return {k: field_map[k] for k in _param_names if k in field_map} + + # Fallback: introspect at call time (for direct callers) + sig = inspect.signature(fn) + kwargs: dict[str, Any] = {} + + for name, param in sig.parameters.items(): + if name in field_map: + kwargs[name] = field_map[name] + elif param.default is inspect.Parameter.empty: + raise TypeError( + f"Function evaluator '{fn.__name__}' has unknown required parameter " + f"'{name}'. Supported: {sorted(_KNOWN_PARAMS)}" + ) + # else: has a default — leave it to Python + + return kwargs + + +def _coerce_result(value: Any, check_name: str) -> CheckResult: + """Convert a function evaluator return value to a ``CheckResult``. + + Accepted return types: + + * ``bool`` — True/False maps directly to pass/fail. + * ``int | float`` — ≥ 0.5 is pass (score is included in reason). + * ``CheckResult`` — returned as-is. + * ``dict`` with ``score`` or ``passed`` key — converted to CheckResult. + """ + if isinstance(value, CheckResult): + return value + + if isinstance(value, bool): + return CheckResult(passed=value, reason="passed" if value else "failed", check_name=check_name) + + if isinstance(value, (int, float)): + passed = value >= 0.5 + return CheckResult(passed=passed, reason=f"score={value:.3f}", check_name=check_name) + + if isinstance(value, dict): + d = cast(dict[str, Any], value) + if "score" in d: + try: + score = float(d["score"]) + except (TypeError, ValueError) as exc: + raise TypeError( + f"Function evaluator '{check_name}' returned dict with non-numeric 'score' value:" + f" {d['score']!r}" + ) from exc + # Honour an explicit 'passed' override; otherwise threshold-based. + passed = bool(d["passed"]) if "passed" in d else score >= float(d.get("threshold", 0.5)) + reason = str(d.get("reason", f"score={score:.3f}")) + return CheckResult(passed=passed, reason=reason, check_name=check_name) + if "passed" in d: + passed_val = d["passed"] + if not isinstance(passed_val, (bool, int)): + raise TypeError( + f"Function evaluator '{check_name}' returned dict with non-boolean 'passed' value: {passed_val!r}" + ) + return CheckResult( + passed=bool(passed_val), + reason=str(d.get("reason", "passed" if passed_val else "failed")), + check_name=check_name, + ) + + value_type_name = type(value).__name__ # type: ignore[reportUnknownMemberType] + msg = ( + f"Function evaluator '{check_name}' returned unsupported type " + f"{value_type_name}. Expected bool, float, dict, or CheckResult." + ) + raise TypeError(msg) + + +@overload +def evaluator(fn: Callable[..., Any], /) -> EvalCheck: ... + + +@overload +def evaluator(*, name: str | None = None) -> Callable[[Callable[..., Any]], EvalCheck]: ... + + +def evaluator( + fn: Callable[..., Any] | None = None, + *, + name: str | None = None, +) -> EvalCheck | Callable[[Callable[..., Any]], EvalCheck]: + """Wrap a plain function as an ``EvalCheck`` for use with ``LocalEvaluator``. + + Works with both sync and async functions. The function's parameter names + determine what data it receives from the ``EvalItem``. Any combination of + the following parameter names is valid: + + * ``query`` — the user query (str) + * ``response`` — the agent response (str) + * ``expected_output`` — expected output for ground-truth comparison (str) + * ``conversation`` — full conversation history (list[Message]) + * ``tools`` — typed tool objects (list[FunctionTool]) + * ``context`` — grounding context (str | None) + + Return ``bool``, ``float`` (≥0.5 = pass), ``dict`` with ``score`` or + ``passed`` key, or ``CheckResult``. + + Can be used as a decorator (with or without arguments) or called directly: + + .. code-block:: python + + # Decorator — no args + @evaluator + def mentions_weather(query: str, response: str) -> bool: + return "weather" in response.lower() + + + # Decorator — with name + @evaluator(name="length_check") + def is_not_too_long(response: str) -> bool: + return len(response) < 2000 + + + # Direct wrapping + check = evaluator(my_scorer, name="my_scorer") + + + # Async function — handled automatically + @evaluator + async def llm_judge(query: str, response: str) -> float: + result = await my_llm_client.score(query, response) + return result.score + + + # Use with LocalEvaluator + local = LocalEvaluator(mentions_weather, is_not_too_long, check, llm_judge) + + Args: + fn: The function to wrap. If omitted, returns a decorator. + name: Display name for the check (defaults to ``fn.__name__``). + """ + + def _wrap(func: Callable[..., Any]) -> EvalCheck: + check_name: str = name or getattr(func, "__name__", None) or "evaluator" + # Cache signature introspection once per wrapped function + sig = inspect.signature(func) + param_names = { + n for n, p in sig.parameters.items() if n in _KNOWN_PARAMS or p.default is inspect.Parameter.empty + } + required_unknown = { + n for n, p in sig.parameters.items() if n not in _KNOWN_PARAMS and p.default is inspect.Parameter.empty + } + if required_unknown: + raise TypeError( + f"Function evaluator '{func.__name__}' has unknown required parameter(s) " + f"{sorted(required_unknown)}. Supported: {sorted(_KNOWN_PARAMS)}" + ) + + async def _check(item: EvalItem) -> CheckResult: + kwargs = _resolve_function_args(func, item, _param_names=param_names) + result = func(**kwargs) + if inspect.isawaitable(result): + result = await result + return _coerce_result(value=result, check_name=check_name) + + _check.__name__ = check_name # type: ignore[attr-defined,assignment] + _check.__doc__ = func.__doc__ + return _check + + # Support @evaluator (no parens) and @evaluator(name="x") + if fn is not None: + return _wrap(fn) + return _wrap + + +# endregion + +# region LocalEvaluator + + +async def _run_check(check_fn: EvalCheck, item: EvalItem) -> CheckResult: + """Run a single check, awaiting the result if it is a coroutine.""" + result = check_fn(item) + if inspect.isawaitable(result): + result = await result + return result + + +class LocalEvaluator: + """Evaluation provider that runs checks locally without API calls. + + Implements the ``Evaluator`` protocol. Each check function is applied + to every item. An item passes only if all checks pass. + + Examples: + Basic usage: + + .. code-block:: python + + from agent_framework import LocalEvaluator, keyword_check, evaluate_agent + + local = LocalEvaluator( + keyword_check("weather"), + tool_called_check("get_weather"), + ) + results = await evaluate_agent(agent=agent, queries=queries, evaluators=local) + + Mixing with cloud evaluators: + + .. code-block:: python + + from agent_framework.foundry import FoundryEvals + + results = await evaluate_agent( + agent=agent, + queries=queries, + evaluators=[local, FoundryEvals(project_client=client, model="gpt-4o")], + ) + """ + + def __init__(self, *checks: EvalCheck): + self.name = "Local" + self._checks = checks + + async def evaluate( + self, + items: Sequence[EvalItem], + *, + eval_name: str = "Local Eval", + ) -> EvalResults: + """Run all checks on each item and return aggregated results. + + An item passes only if every check passes for that item. Per-check + breakdowns are available in ``per_evaluator``. + + Supports both sync and async check functions (from + :func:`evaluator`). + """ + passed = 0 + failed = 0 + per_check: dict[str, dict[str, int]] = {} + failure_reasons: list[str] = [] + result_items: list[EvalItemResult] = [] + + for item_idx, item in enumerate(items): + check_results = await asyncio.gather(*[_run_check(fn, item) for fn in self._checks]) + item_passed = True + item_scores: list[EvalScoreResult] = [] + for result in check_results: + counts = per_check.setdefault(result.check_name, {"passed": 0, "failed": 0, "errored": 0}) + if result.passed: + counts["passed"] += 1 + else: + counts["failed"] += 1 + item_passed = False + failure_reasons.append(f"{result.check_name}: {result.reason}") + item_scores.append( + EvalScoreResult( + name=result.check_name, + score=1.0 if result.passed else 0.0, + passed=result.passed, + sample={"reason": result.reason} if result.reason else None, + ) + ) + + if item_passed: + passed += 1 + else: + failed += 1 + + result_items.append( + EvalItemResult( + item_id=str(item_idx), + status="pass" if item_passed else "fail", + scores=item_scores, + input_text=item.query, + output_text=item.response, + ) + ) + + return EvalResults( + provider=self.name, + eval_id="local", + run_id=eval_name, + status="completed", + result_counts={"passed": passed, "failed": failed, "errored": 0}, + per_evaluator=per_check, + items=result_items, + error="; ".join(failure_reasons) if failure_reasons else None, + ) + + +# endregion + +# region Public orchestration functions + + +async def evaluate_agent( + *, + agent: SupportsAgentRun | None = None, + queries: str | Sequence[str] | None = None, + expected_output: str | Sequence[str] | None = None, + expected_tool_calls: Sequence[ExpectedToolCall] | Sequence[Sequence[ExpectedToolCall]] | None = None, + responses: AgentResponse[Any] | Sequence[AgentResponse[Any]] | None = None, + evaluators: Evaluator | Callable[..., Any] | Sequence[Evaluator | Callable[..., Any]], + eval_name: str | None = None, + context: str | None = None, + conversation_split: ConversationSplitter | None = None, + num_repetitions: int = 1, +) -> list[EvalResults]: + """Run an agent against test queries and evaluate the results. + + The simplest path for evaluating an agent during development. For each + query, runs the agent, converts the interaction to eval format, and + submits to the evaluator(s). + + All sequence parameters (``queries``, ``expected_output``, + ``expected_tool_calls``, ``responses``) accept either a single value + or a list for convenience. + + If ``responses`` is provided, skips running the agent and evaluates those + responses directly — but still extracts tool definitions from the agent. + In this mode ``queries`` is required to construct the conversation. + + Args: + agent: An agent-framework agent instance. + queries: Test query or queries to run the agent against. A single + string is wrapped into a one-element list. Required when + ``responses`` is not provided. + expected_output: Ground-truth expected output(s), one per query. A + single string is wrapped into a one-element list. When provided, + must be the same length as ``queries``. Each value is stamped on + the corresponding ``EvalItem.expected_output`` for evaluators + that compare against a reference answer. + expected_tool_calls: Expected tool call(s), one list per query. A + single flat list of ``ExpectedToolCall`` is wrapped into a + one-element nested list. When provided, must be the same length + as ``queries``. + responses: Pre-existing ``AgentResponse``(s) to evaluate without + running the agent. A single response is wrapped into a one-element + list. When provided, ``queries`` must also be provided to + construct the conversation for evaluation. + evaluators: One or more ``Evaluator`` instances. + eval_name: Display name (defaults to agent name). + context: Optional context for groundedness evaluation. + conversation_split: Split strategy applied to all items, overriding + each evaluator's default. See ``ConversationSplitter``. + num_repetitions: Number of times to run each query (default 1). + When > 1, each query is invoked independently N times to measure + consistency. Results contain all N x len(queries) items. + Ignored when ``responses`` is provided (pre-existing responses + are evaluated as-is). + + Returns: + A list of ``EvalResults``, one per evaluator provider. + + Raises: + ValueError: If neither ``queries`` nor ``responses`` is provided. + + Examples: + Run and evaluate: + + .. code-block:: python + + results = await evaluate_agent( + agent=my_agent, + queries="What's the weather?", + evaluators=evals, + ) + + Evaluate existing responses: + + .. code-block:: python + + response = await agent.run([Message("user", ["What's the weather?"])]) + results = await evaluate_agent( + agent=agent, + responses=response, + queries="What's the weather?", + evaluators=evals, + ) + + With ground-truth expected answers: + + .. code-block:: python + + results = await evaluate_agent( + agent=my_agent, + queries=["What's 2+2?", "Capital of France?"], + expected_output=["4", "Paris"], + evaluators=evals, + ) + + With expected tool calls: + + .. code-block:: python + + results = await evaluate_agent( + agent=my_agent, + queries="What's the weather in NYC?", + expected_tool_calls=[ExpectedToolCall("get_weather", {"location": "NYC"})], + evaluators=evals, + ) + """ + # Normalize singular values to lists + if isinstance(queries, str): + queries = [queries] + if isinstance(expected_output, str): + expected_output = [expected_output] + if isinstance(responses, AgentResponse): + responses = [responses] + if ( + expected_tool_calls is not None + and len(expected_tool_calls) > 0 + and isinstance(expected_tool_calls[0], ExpectedToolCall) + ): + expected_tool_calls = [list(cast(Sequence[ExpectedToolCall], expected_tool_calls))] + + items: list[EvalItem] = [] + + # Validate num_repetitions + if num_repetitions < 1: + raise ValueError(f"num_repetitions must be >= 1, got {num_repetitions}.") + + # Validate expected_output length against queries + if expected_output is not None and queries is not None and len(expected_output) != len(queries): + raise ValueError(f"Got {len(queries)} queries but {len(expected_output)} expected_output values.") + + # Validate expected_tool_calls length against queries + if expected_tool_calls is not None and queries is not None and len(expected_tool_calls) != len(queries): + raise ValueError(f"Got {len(queries)} queries but {len(expected_tool_calls)} expected_tool_calls lists.") + + if responses is not None: + # Evaluate pre-existing responses (don't run the agent) + resp_list = list(responses) + + if queries is not None: + query_list = list(queries) + if len(query_list) != len(resp_list): + raise ValueError(f"Got {len(query_list)} queries but {len(resp_list)} responses.") + for q, r in zip(query_list, resp_list): + items.append( + AgentEvalConverter.to_eval_item( + query=q, + response=r, + agent=agent, + context=context, + ) + ) + else: + raise ValueError( + "Provide 'queries' alongside 'responses' so the conversation " + "can be constructed for evaluation. For Responses API " + "evaluation by response ID, use evaluate_traces(response_ids=...) from " + "the azure-ai package." + ) + elif queries is not None and agent is not None: + # Run the agent against test queries, with repetitions + for _rep in range(num_repetitions): + for query in queries: + response = await agent.run([Message("user", [query])]) + items.append( + AgentEvalConverter.to_eval_item( + query=query, + response=response, + agent=agent, + context=context, + ) + ) + elif queries is not None and agent is None: + raise ValueError( + "Provide 'agent' when using 'queries' to run the agent. " + "To evaluate pre-existing responses without an agent, use 'responses=' instead." + ) + else: + raise ValueError("Provide either 'queries' (with 'agent') or 'responses' (or both).") + + # Stamp expected output values on items (repeated across all repetitions) + if expected_output is not None: + query_count = len(expected_output) + for i, item in enumerate(items): + item.expected_output = expected_output[i % query_count] + + # Stamp expected tool calls on items (repeated across all repetitions) + if expected_tool_calls is not None: + # After normalization, expected_tool_calls is Sequence[Sequence[ExpectedToolCall]] + tc_list = cast(Sequence[Sequence[ExpectedToolCall]], expected_tool_calls) + query_count = len(tc_list) + for i, item in enumerate(items): + item.expected_tool_calls = list(tc_list[i % query_count]) + + # Stamp split strategy on items so evaluators respect it + if conversation_split is not None: + for item in items: + item.split_strategy = conversation_split + + name = eval_name or f"Eval: {getattr(agent, 'name', None) or getattr(agent, 'id', 'agent') if agent else 'agent'}" + return await _run_evaluators(evaluators, items, eval_name=name) + + +async def evaluate_workflow( + *, + workflow: Workflow, + workflow_result: WorkflowRunResult | None = None, + queries: str | Sequence[str] | None = None, + evaluators: Evaluator | Callable[..., Any] | Sequence[Evaluator | Callable[..., Any]], + eval_name: str | None = None, + include_overall: bool = True, + include_per_agent: bool = True, + conversation_split: ConversationSplitter | None = None, + num_repetitions: int = 1, +) -> list[EvalResults]: + """Evaluate a multi-agent workflow with per-agent breakdown. + + Evaluates each sub-agent individually and (optionally) the workflow's + overall output. Returns one ``EvalResults`` per evaluator provider, each + with per-agent breakdowns in ``sub_results``. + + **Two modes:** + + - **Post-hoc**: Pass ``workflow_result`` from a previous + ``workflow.run()`` call. + - **Run + evaluate**: Pass ``queries`` and the workflow will be run + against each query, then evaluated. + + Args: + workflow: The workflow instance. + workflow_result: A completed ``WorkflowRunResult``. + queries: Test queries to run through the workflow. + evaluators: One or more ``Evaluator`` instances. + eval_name: Display name for the evaluation. + include_overall: Whether to evaluate the workflow's final output. + include_per_agent: Whether to evaluate each sub-agent individually. + conversation_split: Split strategy applied to all items, overriding + each evaluator's default. See ``ConversationSplitter``. + num_repetitions: Number of times to run each query (default 1). + When > 1, each query is run independently N times. + Ignored when ``workflow_result`` is provided. + + Returns: + A list of ``EvalResults``, one per evaluator provider. + + Example: + + .. code-block:: python + + from agent_framework.foundry import FoundryEvals + + evals = FoundryEvals(project_client=client, model="gpt-4o") + result = await workflow.run("Plan a trip to Paris") + + eval_results = await evaluate_workflow( + workflow=workflow, + workflow_result=result, + evaluators=evals, + ) + for r in eval_results: + print(f"{r.provider}:") + for name, sub in r.sub_results.items(): + print(f" {name}: {sub.passed}/{sub.total}") + """ + from ._workflows._workflow import WorkflowRunResult as WRR + + # Normalize singular query to list + if isinstance(queries, str): + queries = [queries] + + if workflow_result is None and queries is None: + raise ValueError("Provide either 'workflow_result' or 'queries'.") + + if num_repetitions < 1: + raise ValueError(f"num_repetitions must be >= 1, got {num_repetitions}.") + + wf_name = eval_name or f"Workflow Eval: {workflow.__class__.__name__}" + evaluator_list = _resolve_evaluators(evaluators) + + # Collect per-agent data and overall items + all_agent_data: list[_AgentEvalData] = [] + overall_items: list[EvalItem] = [] + + if queries is not None: + results_list: list[WRR] = [] + for _rep in range(num_repetitions): + for q in queries: + result = await workflow.run(q) + if not isinstance(result, WRR): + raise TypeError(f"Expected WorkflowRunResult from workflow.run(), got {type(result).__name__}.") + results_list.append(result) + all_agent_data.extend(_extract_agent_eval_data(result, workflow)) + if include_overall: + overall_item = _build_overall_item(q, result) + if overall_item: + overall_items.append(overall_item) + else: + assert workflow_result is not None # noqa: S101 # nosec B101 + all_agent_data = _extract_agent_eval_data(workflow_result, workflow) + if include_overall: + original_query = _extract_overall_query(workflow_result) + if original_query: + overall_item = _build_overall_item(original_query, workflow_result) + if overall_item: + overall_items.append(overall_item) + + # Group agent data by executor ID + agents_by_id: dict[str, list[_AgentEvalData]] = {} + if include_per_agent and all_agent_data: + for ad in all_agent_data: + agents_by_id.setdefault(ad["executor_id"], []).append(ad) + + # Build per-agent items once (shared across providers). + agent_items_by_id: dict[str, list[EvalItem]] = {} + for executor_id, agent_data_list in agents_by_id.items(): + agent_items_by_id[executor_id] = [ + AgentEvalConverter.to_eval_item( + query=ad["query"], + response=ad["response"], + agent=ad["agent"], + ) + for ad in agent_data_list + ] + + if not agent_items_by_id and not overall_items: + raise ValueError( + "No agent executor data found in the workflow result. Ensure the workflow uses AgentExecutor-based agents." + ) + + # Stamp split strategy on all items so evaluators respect it + if conversation_split is not None: + for items in agent_items_by_id.values(): + for item in items: + item.split_strategy = conversation_split + for item in overall_items: + item.split_strategy = conversation_split + + # Run each provider, building per-agent sub_results for each + all_results: list[EvalResults] = [] + for ev in evaluator_list: + suffix = f" ({ev.name})" if len(evaluator_list) > 1 else "" + sub_results: dict[str, EvalResults] = {} + + # Per-agent evals + for executor_id, items in agent_items_by_id.items(): + agent_result = await ev.evaluate(items, eval_name=f"{wf_name} — {executor_id}{suffix}") + sub_results[executor_id] = agent_result + + # Overall eval + if include_overall and overall_items: + overall_result = await ev.evaluate(overall_items, eval_name=f"{wf_name} — overall{suffix}") + elif sub_results: + # Aggregate from sub-results + total_passed = sum(s.passed for s in sub_results.values()) + total_failed = sum(s.failed for s in sub_results.values()) + all_completed = all(s.status == "completed" for s in sub_results.values()) + overall_result = EvalResults( + provider=ev.name, + eval_id="aggregate", + run_id="aggregate", + status="completed" if all_completed else "partial", + result_counts={ + "passed": total_passed, + "failed": total_failed, + }, + ) + else: + raise ValueError( + "No agent executor data found in the workflow result. " + "Ensure the workflow uses AgentExecutor-based agents." + ) + + overall_result.sub_results = sub_results + all_results.append(overall_result) + + return all_results + + +# endregion + +# region Internal helpers + + +def _build_overall_item( + query: str, + workflow_result: WorkflowRunResult, +) -> EvalItem | None: + """Build an EvalItem for the overall workflow output.""" + outputs = workflow_result.get_outputs() + if not outputs: + return None + + final_output: Any = outputs[-1] + overall_response: AgentResponse[None] + if isinstance(final_output, list) and final_output and isinstance(final_output[0], Message): + msgs: list[Message] = [m for m in cast(list[Any], final_output) if isinstance(m, Message)] # type: ignore[redundant-cast] + response_text = " ".join(str(m.text) for m in msgs if m.role == "assistant") + overall_response = AgentResponse(messages=[Message("assistant", [response_text])]) + elif isinstance(final_output, AgentResponse): + overall_response = cast(AgentResponse[None], final_output) + else: + overall_response = AgentResponse( + messages=[Message("assistant", [str(final_output)])] # type: ignore[reportUnknownArgumentType] + ) + + return AgentEvalConverter.to_eval_item(query=query, response=overall_response) + + +def _resolve_evaluators( + evaluators: Evaluator | Callable[..., Any] | Sequence[Evaluator | Callable[..., Any]], +) -> list[Evaluator]: + """Normalize evaluators into a list of concrete ``Evaluator`` instances. + + Bare callables (``EvalCheck`` functions, ``@evaluator`` decorated) are + collected and wrapped in a single ``LocalEvaluator``. + """ + raw_list: list[Any] = ( + [evaluators] if isinstance(evaluators, Evaluator) or callable(evaluators) else list(evaluators) + ) + + resolved: list[Evaluator] = [] + pending_checks: list[Callable[..., Any]] = [] + + for item in raw_list: + if isinstance(item, Evaluator): + if pending_checks: + resolved.append(LocalEvaluator(*pending_checks)) + pending_checks = [] + resolved.append(item) + elif callable(item): + pending_checks.append(item) + else: + raise TypeError(f"Expected an Evaluator or callable, got {type(item).__name__}") + + if pending_checks: + resolved.append(LocalEvaluator(*pending_checks)) + + return resolved + + +async def _run_evaluators( + evaluators: Evaluator | Callable[..., Any] | Sequence[Evaluator | Callable[..., Any]], + items: Sequence[EvalItem], + *, + eval_name: str, +) -> list[EvalResults]: + """Run one or more evaluators and return a result per provider. + + Bare ``EvalCheck`` callables (including ``@evaluator`` decorated + functions and helpers like ``keyword_check``) are auto-wrapped in a + ``LocalEvaluator`` so they can be passed directly in the evaluators list. + """ + evaluator_list = _resolve_evaluators(evaluators) + + async def _run_single_evaluator( + ev: Evaluator, + eval_items: Sequence[EvalItem], + name: str, + suffix: str, + ) -> EvalResults: + return await ev.evaluate(eval_items, eval_name=f"{name}{suffix}") + + results = await asyncio.gather(*[ + _run_single_evaluator(ev, items, eval_name, f" ({ev.name})" if len(evaluator_list) > 1 else "") + for ev in evaluator_list + ]) + return list(results) + + +# endregion diff --git a/python/packages/core/agent_framework/foundry/__init__.py b/python/packages/core/agent_framework/foundry/__init__.py index b8092909b4..0ebf0a9389 100644 --- a/python/packages/core/agent_framework/foundry/__init__.py +++ b/python/packages/core/agent_framework/foundry/__init__.py @@ -12,6 +12,7 @@ _IMPORTS: dict[str, tuple[str, str]] = { "FoundryAgent": ("agent_framework_foundry", "agent-framework-foundry"), "FoundryChatClient": ("agent_framework_foundry", "agent-framework-foundry"), "FoundryChatOptions": ("agent_framework_foundry", "agent-framework-foundry"), + "FoundryEvals": ("agent_framework_foundry", "agent-framework-foundry"), "FoundryMemoryProvider": ("agent_framework_foundry", "agent-framework-foundry"), "FoundryLocalChatOptions": ("agent_framework_foundry_local", "agent-framework-foundry-local"), "FoundryLocalClient": ("agent_framework_foundry_local", "agent-framework-foundry-local"), @@ -19,6 +20,8 @@ _IMPORTS: dict[str, tuple[str, str]] = { "RawFoundryAgent": ("agent_framework_foundry", "agent-framework-foundry"), "RawFoundryAgentChatClient": ("agent_framework_foundry", "agent-framework-foundry"), "RawFoundryChatClient": ("agent_framework_foundry", "agent-framework-foundry"), + "evaluate_foundry_target": ("agent_framework_foundry", "agent-framework-foundry"), + "evaluate_traces": ("agent_framework_foundry", "agent-framework-foundry"), } diff --git a/python/packages/core/agent_framework/foundry/__init__.pyi b/python/packages/core/agent_framework/foundry/__init__.pyi index 22c0b38b06..534b7fa5bc 100644 --- a/python/packages/core/agent_framework/foundry/__init__.pyi +++ b/python/packages/core/agent_framework/foundry/__init__.pyi @@ -7,10 +7,13 @@ from agent_framework_foundry import ( FoundryAgent, FoundryChatClient, FoundryChatOptions, + FoundryEvals, FoundryMemoryProvider, RawFoundryAgent, RawFoundryAgentChatClient, RawFoundryChatClient, + evaluate_foundry_target, + evaluate_traces, ) from agent_framework_foundry_local import ( FoundryLocalChatOptions, @@ -22,6 +25,7 @@ __all__ = [ "FoundryAgent", "FoundryChatClient", "FoundryChatOptions", + "FoundryEvals", "FoundryLocalChatOptions", "FoundryLocalClient", "FoundryLocalSettings", @@ -29,4 +33,6 @@ __all__ = [ "RawFoundryAgent", "RawFoundryAgentChatClient", "RawFoundryChatClient", + "evaluate_foundry_target", + "evaluate_traces", ] diff --git a/python/packages/core/pyproject.toml b/python/packages/core/pyproject.toml index caeebc5319..f002567381 100644 --- a/python/packages/core/pyproject.toml +++ b/python/packages/core/pyproject.toml @@ -4,7 +4,7 @@ description = "Microsoft Agent Framework for building AI Agents with Python. Thi authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0rc5" +version = "1.0.0rc6" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py new file mode 100644 index 0000000000..96b0e1a391 --- /dev/null +++ b/python/packages/core/tests/core/test_local_eval.py @@ -0,0 +1,1028 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for evaluator checks and LocalEvaluator.""" + +from __future__ import annotations + +import inspect + +import pytest + +from agent_framework._evaluation import ( + CheckResult, + EvalItem, + ExpectedToolCall, + LocalEvaluator, + _coerce_result, + evaluator, + keyword_check, + tool_call_args_match, + tool_called_check, + tool_calls_present, +) +from agent_framework._types import Content, Message + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_item( + query: str = "What's the weather in Paris?", + response: str = "It's sunny and 75°F", + expected_output: str | None = None, + conversation: list | None = None, + tools: list | None = None, + context: str | None = None, +) -> EvalItem: + if conversation is None: + conversation = [Message("user", [query]), Message("assistant", [response])] + return EvalItem( + conversation=conversation, + expected_output=expected_output, + tools=tools, + context=context, + ) + + +# --------------------------------------------------------------------------- +# Tier 1: (query, response) -> result +# --------------------------------------------------------------------------- + + +class TestTier1SimpleChecks: + @pytest.mark.asyncio + async def test_bool_return_true(self): + @evaluator + def has_temperature(query: str, response: str) -> bool: + return "°F" in response + + result = await has_temperature(_make_item()) + assert result.passed is True + assert result.check_name == "has_temperature" + + @pytest.mark.asyncio + async def test_bool_return_false(self): + @evaluator + def has_celsius(query: str, response: str) -> bool: + return "°C" in response + + result = await has_celsius(_make_item()) + assert result.passed is False + + @pytest.mark.asyncio + async def test_float_return_passing(self): + @evaluator + def length_score(response: str) -> float: + return min(len(response) / 10, 1.0) + + result = await length_score(_make_item()) + assert result.passed is True + assert "score=" in result.reason + + @pytest.mark.asyncio + async def test_float_return_failing(self): + @evaluator + def always_low(response: str) -> float: + return 0.1 + + result = await always_low(_make_item()) + assert result.passed is False + + @pytest.mark.asyncio + async def test_response_only(self): + """Function with only 'response' param should work.""" + + @evaluator + def is_short(response: str) -> bool: + return len(response) < 1000 + + result = await is_short(_make_item()) + assert result.passed is True + + @pytest.mark.asyncio + async def test_query_only(self): + """Function with only 'query' param should work.""" + + @evaluator + def is_question(query: str) -> bool: + return "?" in query + + result = await is_question(_make_item()) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Tier 2: (query, response, expected_output) -> result +# --------------------------------------------------------------------------- + + +class TestTier2GroundTruth: + @pytest.mark.asyncio + async def test_exact_match(self): + @evaluator + def exact_match(response: str, expected_output: str) -> bool: + return response.strip() == expected_output.strip() + + item = _make_item(response="42", expected_output="42") + assert (await exact_match(item)).passed is True + + item2 = _make_item(response="43", expected_output="42") + assert (await exact_match(item2)).passed is False + + @pytest.mark.asyncio + async def test_expected_output_defaults_to_empty(self): + """When expected_output is None on the item, it should be passed as ''.""" + + @evaluator + def check_expected(expected_output: str) -> bool: + return expected_output == "" + + result = await check_expected(_make_item(expected_output=None)) + assert result.passed is True + + @pytest.mark.asyncio + async def test_similarity_score(self): + @evaluator + def word_overlap(response: str, expected_output: str) -> float: + r_words = set(response.lower().split()) + e_words = set(expected_output.lower().split()) + if not e_words: + return 1.0 + return len(r_words & e_words) / len(e_words) + + item = _make_item(response="sunny warm day", expected_output="warm sunny afternoon") + result = await word_overlap(item) + assert result.passed is True # 2/3 overlap ≥ 0.5 + + +# --------------------------------------------------------------------------- +# Tier 3: full context (conversation, tools, context) +# --------------------------------------------------------------------------- + + +class TestTier3FullContext: + @pytest.mark.asyncio + async def test_conversation_access(self): + @evaluator + def multi_turn(query: str, response: str, *, conversation: list) -> bool: + return len(conversation) >= 2 + + item = _make_item(conversation=[Message("user", []), Message("assistant", [])]) + assert (await multi_turn(item)).passed is True + + item2 = _make_item(conversation=[Message("user", [])]) + assert (await multi_turn(item2)).passed is False + + @pytest.mark.asyncio + async def test_tools_access(self): + @evaluator + def has_tools(tools: list) -> bool: + return len(tools) > 0 + + mock_tool = type( + "MockTool", + (), + {"name": "get_weather", "description": "Get weather", "parameters": lambda self: {}}, + )() + item = _make_item(tools=[mock_tool]) + assert (await has_tools(item)).passed is True + + @pytest.mark.asyncio + async def test_context_access(self): + @evaluator + def grounded(response: str, context: str) -> bool: + if not context: + return True + return any(word in response.lower() for word in context.lower().split()) + + item = _make_item(response="It's sunny", context="sunny warm") + assert (await grounded(item)).passed is True + + @pytest.mark.asyncio + async def test_all_params(self): + @evaluator + def full_check( + query: str, + response: str, + expected_output: str, + conversation: list, + tools: list, + context: str, + ) -> bool: + return all([query, response, expected_output is not None, isinstance(conversation, list)]) + + item = _make_item(expected_output="foo", context="bar") + assert (await full_check(item)).passed is True + + +# --------------------------------------------------------------------------- +# Return type coercion +# --------------------------------------------------------------------------- + + +class TestReturnTypeCoercion: + @pytest.mark.asyncio + async def test_dict_with_score(self): + @evaluator + def scored(response: str) -> dict: + return {"score": 0.9, "reason": "good answer"} + + result = await scored(_make_item()) + assert result.passed is True + assert result.reason == "good answer" + + @pytest.mark.asyncio + async def test_dict_with_score_below_threshold(self): + @evaluator + def low_scored(response: str) -> dict: + return {"score": 0.3} + + result = await low_scored(_make_item()) + assert result.passed is False + + @pytest.mark.asyncio + async def test_dict_with_custom_threshold(self): + @evaluator + def custom_threshold(response: str) -> dict: + return {"score": 0.3, "threshold": 0.2} + + result = await custom_threshold(_make_item()) + assert result.passed is True + + @pytest.mark.asyncio + async def test_dict_with_passed(self): + @evaluator + def explicit_pass(response: str) -> dict: + return {"passed": True, "reason": "all good"} + + result = await explicit_pass(_make_item()) + assert result.passed is True + assert result.reason == "all good" + + @pytest.mark.asyncio + async def test_check_result_passthrough(self): + @evaluator + def returns_check_result(response: str) -> CheckResult: + return CheckResult(True, "direct result", "custom") + + result = await returns_check_result(_make_item()) + assert result.passed is True + assert result.reason == "direct result" + assert result.check_name == "custom" + + @pytest.mark.asyncio + async def test_unsupported_return_type(self): + @evaluator + def bad_return(response: str) -> str: + return "oops" + + with pytest.raises(TypeError, match="unsupported type"): + await bad_return(_make_item()) + + @pytest.mark.asyncio + async def test_int_return(self): + @evaluator + def int_score(response: str) -> int: + return 1 + + result = await int_score(_make_item()) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Decorator variants +# --------------------------------------------------------------------------- + + +class TestDecoratorVariants: + @pytest.mark.asyncio + async def test_decorator_no_parens(self): + @evaluator + def my_check(response: str) -> bool: + return True + + assert (await my_check(_make_item())).passed is True + + @pytest.mark.asyncio + async def test_decorator_with_name(self): + @evaluator(name="custom_name") + def my_check(response: str) -> bool: + return True + + assert my_check.__name__ == "custom_name" + result = await my_check(_make_item()) + assert result.check_name == "custom_name" + + @pytest.mark.asyncio + async def test_direct_call(self): + def raw_fn(query: str, response: str) -> bool: + return len(response) > 0 + + check = evaluator(raw_fn, name="direct") + result = await check(_make_item()) + assert result.passed is True + assert result.check_name == "direct" + + +# --------------------------------------------------------------------------- +# Error handling +# --------------------------------------------------------------------------- + + +class TestErrorHandling: + @pytest.mark.asyncio + async def test_unknown_required_param_raises(self): + with pytest.raises(TypeError, match="unknown required parameter"): + + @evaluator + def bad_params(query: str, unknown_param: str) -> bool: + return True + + @pytest.mark.asyncio + async def test_unknown_optional_param_ok(self): + @evaluator + def optional_unknown(query: str, foo: str = "default") -> bool: + return foo == "default" + + result = await optional_unknown(_make_item()) + assert result.passed is True + + @pytest.mark.asyncio + async def test_async_function_works_with_evaluator(self): + """Using an async function with @evaluator should work.""" + + @evaluator + async def async_fn(response: str) -> bool: + return True + + result = async_fn(_make_item()) + # Should return an awaitable + assert inspect.isawaitable(result) + check_result = await result + assert check_result.passed is True + + +# --------------------------------------------------------------------------- +# Integration with LocalEvaluator +# --------------------------------------------------------------------------- + + +class TestLocalEvaluatorIntegration: + @pytest.mark.asyncio + async def test_mixed_checks(self): + """Function evaluators mix with built-in checks in LocalEvaluator.""" + + @evaluator + def length_ok(response: str) -> bool: + return len(response) > 5 + + local = LocalEvaluator( + keyword_check("sunny"), + length_ok, + ) + items = [_make_item()] + results = await local.evaluate(items, eval_name="mixed test") + + assert results.status == "completed" + assert results.result_counts["passed"] == 1 + assert results.result_counts["failed"] == 0 + + @pytest.mark.asyncio + async def test_evaluator_failure_counted(self): + @evaluator + def always_fail(response: str) -> bool: + return False + + local = LocalEvaluator(always_fail) + results = await local.evaluate([_make_item()]) + + assert results.result_counts["failed"] == 1 + + @pytest.mark.asyncio + async def test_multiple_evaluators(self): + @evaluator + def check_a(response: str) -> float: + return 0.9 + + @evaluator + def check_b(query: str, response: str, expected_output: str) -> bool: + return True + + @evaluator(name="check_c") + def check_c(response: str, conversation: list) -> dict: + return {"score": 0.8, "reason": "looks good"} + + local = LocalEvaluator(check_a, check_b, check_c) + results = await local.evaluate([_make_item(expected_output="test")]) + + assert results.result_counts["passed"] == 1 + assert "check_a" in results.per_evaluator + assert "check_b" in results.per_evaluator + assert "check_c" in results.per_evaluator + + +# --------------------------------------------------------------------------- +# Async evaluator (via @evaluator which handles async automatically) +# --------------------------------------------------------------------------- + + +class TestAsyncFunctionEvaluator: + @pytest.mark.asyncio + async def test_async_evaluator_in_local(self): + @evaluator + async def async_check(query: str, response: str) -> bool: + return len(response) > 0 + + local = LocalEvaluator(async_check) + results = await local.evaluate([_make_item()]) + assert results.result_counts["passed"] == 1 + + @pytest.mark.asyncio + async def test_async_with_name(self): + @evaluator(name="named_async") + async def my_async(response: str) -> float: + return 0.75 + + result = await my_async(_make_item()) + assert result.passed is True + assert result.check_name == "named_async" + + +# --------------------------------------------------------------------------- +# Auto-wrapping bare checks in evaluate_agent +# --------------------------------------------------------------------------- + + +class TestAutoWrapEvalChecks: + @pytest.mark.asyncio + async def test_bare_check_in_evaluators_list(self): + """Bare EvalCheck callables are auto-wrapped in LocalEvaluator.""" + from agent_framework._evaluation import _run_evaluators + + @evaluator + def is_long(response: str) -> bool: + return len(response.split()) > 2 + + items = [_make_item(response="It is sunny and warm today")] + results = await _run_evaluators(is_long, items, eval_name="test") + assert len(results) == 1 + assert results[0].result_counts["passed"] == 1 + + @pytest.mark.asyncio + async def test_mixed_evaluators_and_checks(self): + """Mix of Evaluator instances and bare checks works.""" + from agent_framework._evaluation import _run_evaluators + + @evaluator + def has_words(response: str) -> bool: + return len(response.split()) > 0 + + local = LocalEvaluator(keyword_check("sunny")) + + items = [_make_item(response="It is sunny")] + results = await _run_evaluators([local, has_words], items, eval_name="test") + assert len(results) == 2 + assert all(r.result_counts["passed"] == 1 for r in results) + + @pytest.mark.asyncio + async def test_adjacent_checks_grouped(self): + """Adjacent bare checks are grouped into a single LocalEvaluator.""" + from agent_framework._evaluation import _run_evaluators + + @evaluator + def check_a(response: str) -> bool: + return True + + @evaluator + def check_b(response: str) -> bool: + return True + + items = [_make_item()] + results = await _run_evaluators([check_a, check_b], items, eval_name="test") + # Two adjacent checks → one LocalEvaluator → one result + assert len(results) == 1 + assert results[0].result_counts["passed"] == 1 + + +# --------------------------------------------------------------------------- +# Expected Tool Calls +# --------------------------------------------------------------------------- + + +def _make_tool_call_item( + calls: list[tuple[str, dict | None]], + expected: list[ExpectedToolCall] | None = None, +) -> EvalItem: + """Build an EvalItem with tool calls in the conversation.""" + msgs: list[Message] = [Message("user", ["Do something"])] + for name, args in calls: + msgs.append(Message("assistant", [Content.from_function_call("call_" + name, name, arguments=args)])) + msgs.append(Message("assistant", ["Done"])) + return EvalItem(conversation=msgs, expected_tool_calls=expected) + + +class TestExpectedToolCallType: + def test_name_only(self): + tc = ExpectedToolCall("get_weather") + assert tc.name == "get_weather" + assert tc.arguments is None + + def test_name_and_args(self): + tc = ExpectedToolCall("get_weather", {"location": "NYC"}) + assert tc.name == "get_weather" + assert tc.arguments == {"location": "NYC"} + + +class TestToolCallsPresent: + def test_all_present(self): + item = _make_tool_call_item( + calls=[("get_weather", None), ("get_news", None)], + expected=[ExpectedToolCall("get_weather"), ExpectedToolCall("get_news")], + ) + result = tool_calls_present(item) + assert result.passed is True + assert result.check_name == "tool_calls_present" + + def test_missing_tool(self): + item = _make_tool_call_item( + calls=[("get_weather", None)], + expected=[ExpectedToolCall("get_weather"), ExpectedToolCall("get_news")], + ) + result = tool_calls_present(item) + assert result.passed is False + assert "get_news" in result.reason + + def test_extras_ok(self): + item = _make_tool_call_item( + calls=[("get_weather", None), ("get_news", None), ("get_stock", None)], + expected=[ExpectedToolCall("get_weather")], + ) + result = tool_calls_present(item) + assert result.passed is True + + def test_no_expected(self): + item = _make_tool_call_item(calls=[("get_weather", None)]) + result = tool_calls_present(item) + assert result.passed is True + assert "No expected" in result.reason + + +class TestToolCallArgsMatch: + def test_name_only_match(self): + item = _make_tool_call_item( + calls=[("get_weather", {"location": "NYC"})], + expected=[ExpectedToolCall("get_weather")], + ) + result = tool_call_args_match(item) + assert result.passed is True + + def test_args_exact_match(self): + item = _make_tool_call_item( + calls=[("get_weather", {"location": "NYC", "units": "fahrenheit"})], + expected=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + # Subset match — extra "units" key is OK + result = tool_call_args_match(item) + assert result.passed is True + + def test_args_mismatch(self): + item = _make_tool_call_item( + calls=[("get_weather", {"location": "LA"})], + expected=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + result = tool_call_args_match(item) + assert result.passed is False + assert "args mismatch" in result.reason + + def test_tool_not_called(self): + item = _make_tool_call_item( + calls=[("get_news", None)], + expected=[ExpectedToolCall("get_weather", {"location": "NYC"})], + ) + result = tool_call_args_match(item) + assert result.passed is False + assert "not called" in result.reason + + def test_multiple_expected(self): + item = _make_tool_call_item( + calls=[ + ("get_weather", {"location": "NYC"}), + ("book_flight", {"destination": "LA", "date": "tomorrow"}), + ], + expected=[ + ExpectedToolCall("get_weather", {"location": "NYC"}), + ExpectedToolCall("book_flight", {"destination": "LA"}), + ], + ) + result = tool_call_args_match(item) + assert result.passed is True + + def test_no_expected(self): + item = _make_tool_call_item(calls=[("get_weather", None)]) + result = tool_call_args_match(item) + assert result.passed is True + + +class TestExpectedToolCallsFieldInjection: + """Test that @evaluator can receive expected_tool_calls via parameter injection.""" + + @pytest.mark.asyncio + async def test_injection(self): + @evaluator + def check_tools(expected_tool_calls: list) -> bool: + return len(expected_tool_calls) == 2 + + item = _make_tool_call_item( + calls=[], + expected=[ExpectedToolCall("a"), ExpectedToolCall("b")], + ) + result = await check_tools(item) + assert result.passed is True + + @pytest.mark.asyncio + async def test_injection_empty_default(self): + @evaluator + def check_tools(expected_tool_calls: list) -> bool: + return len(expected_tool_calls) == 0 + + item = _make_tool_call_item(calls=[]) + result = await check_tools(item) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Per-item results (auditing) +# --------------------------------------------------------------------------- + + +class TestPerItemResults: + """LocalEvaluator should produce per-item EvalItemResult with query/response.""" + + @pytest.mark.asyncio + async def test_items_populated_with_query_and_response(self): + @evaluator + def is_sunny(response: str) -> bool: + return "sunny" in response.lower() + + item = _make_item(query="Weather?", response="It's sunny!") + local = LocalEvaluator(is_sunny) + results = await local.evaluate([item]) + + assert len(results.items) == 1 + ri = results.items[0] + assert ri.item_id == "0" + assert ri.status == "pass" + assert ri.input_text == "Weather?" + assert ri.output_text == "It's sunny!" + assert len(ri.scores) == 1 + assert ri.scores[0].name == "is_sunny" + assert ri.scores[0].passed is True + + @pytest.mark.asyncio + async def test_items_populated_on_failure(self): + @evaluator + def always_fail(response: str) -> bool: + return False + + item = _make_item(query="Hello", response="World") + local = LocalEvaluator(always_fail) + results = await local.evaluate([item]) + + assert len(results.items) == 1 + ri = results.items[0] + assert ri.status == "fail" + assert ri.input_text == "Hello" + assert ri.output_text == "World" + assert ri.scores[0].passed is False + assert ri.scores[0].score == 0.0 + + @pytest.mark.asyncio + async def test_multiple_items_indexed(self): + @evaluator + def pass_all(response: str) -> bool: + return True + + items = [ + _make_item(query="Q1", response="R1"), + _make_item(query="Q2", response="R2"), + ] + local = LocalEvaluator(pass_all) + results = await local.evaluate(items) + + assert len(results.items) == 2 + assert results.items[0].item_id == "0" + assert results.items[0].input_text == "Q1" + assert results.items[0].output_text == "R1" + assert results.items[1].item_id == "1" + assert results.items[1].input_text == "Q2" + assert results.items[1].output_text == "R2" + + +# --------------------------------------------------------------------------- +# num_repetitions validation +# --------------------------------------------------------------------------- + + +class TestNumRepetitions: + """Tests for the num_repetitions parameter on evaluate_agent.""" + + @pytest.mark.asyncio + async def test_num_repetitions_validation_rejects_zero(self): + from agent_framework._evaluation import evaluate_agent + + with pytest.raises(ValueError, match="num_repetitions must be >= 1"): + await evaluate_agent( + queries=["Hello"], + evaluators=LocalEvaluator(keyword_check("hello")), + num_repetitions=0, + ) + + @pytest.mark.asyncio + async def test_num_repetitions_validation_rejects_negative(self): + from agent_framework._evaluation import evaluate_agent + + with pytest.raises(ValueError, match="num_repetitions must be >= 1"): + await evaluate_agent( + queries=["Hello"], + evaluators=LocalEvaluator(keyword_check("hello")), + num_repetitions=-1, + ) + + @pytest.mark.asyncio + async def test_num_repetitions_multiplies_items(self): + """num_repetitions=2 produces 2× the eval items.""" + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse, Message + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock(return_value=AgentResponse(messages=[Message("assistant", ["reply"])])) + + results = await evaluate_agent( + agent=mock_agent, + queries=["Q1", "Q2"], + evaluators=LocalEvaluator(keyword_check("reply")), + num_repetitions=2, + ) + # 2 queries × 2 reps = 4 items + assert results[0].total == 4 + assert mock_agent.run.call_count == 4 + + @pytest.mark.asyncio + async def test_num_repetitions_with_expected_output(self): + """num_repetitions > 1 correctly stamps expected_output via modulo.""" + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse, Message + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock(return_value=AgentResponse(messages=[Message("assistant", ["reply"])])) + + @evaluator + def check_expected(response: str, expected_output: str) -> dict: + return {"passed": expected_output in ("A", "B"), "reason": f"expected={expected_output}"} + + results = await evaluate_agent( + agent=mock_agent, + queries=["Q1", "Q2"], + expected_output=["A", "B"], + evaluators=LocalEvaluator(check_expected), + num_repetitions=2, + ) + # 2 queries × 2 reps = 4 items, all should pass + assert results[0].total == 4 + assert results[0].passed == 4 + + @pytest.mark.asyncio + async def test_num_repetitions_with_expected_tool_calls(self): + """num_repetitions > 1 correctly stamps expected_tool_calls via modulo.""" + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse, Content, Message + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock( + return_value=AgentResponse( + messages=[ + Message( + "assistant", + [Content.from_function_call("c1", "get_weather", arguments={"location": "NYC"})], + ), + Message("tool", [Content.from_function_result("c1", result="Sunny")]), + Message("assistant", ["It's sunny"]), + ] + ) + ) + + results = await evaluate_agent( + agent=mock_agent, + queries=["Q1"], + expected_tool_calls=[[ExpectedToolCall("get_weather")]], + evaluators=LocalEvaluator(tool_calls_present), + num_repetitions=2, + ) + # 1 query × 2 reps = 2 items + assert results[0].total == 2 + assert results[0].passed == 2 + + +# --------------------------------------------------------------------------- +# r3 review: additional test coverage +# --------------------------------------------------------------------------- + + +class TestToolCalledCheckModeAny: + """Tests for tool_called_check with mode='any'.""" + + async def test_any_mode_one_tool_called(self): + """mode='any' passes when at least one expected tool is called.""" + item = _make_item( + conversation=[ + Message("user", ["Do something"]), + Message("assistant", [Content.from_function_call("c1", "tool_a", arguments={})]), + Message("tool", [Content.from_function_result("c1", result="ok")]), + Message("assistant", ["Done"]), + ] + ) + check = tool_called_check("tool_a", "tool_b", mode="any") + result = check(item) + assert result.passed is True + + async def test_any_mode_none_called(self): + """mode='any' fails when no expected tools are called.""" + item = _make_item( + conversation=[ + Message("user", ["Do something"]), + Message("assistant", ["I can't use tools"]), + ] + ) + check = tool_called_check("tool_a", "tool_b", mode="any") + result = check(item) + assert result.passed is False + assert "None of expected tools" in result.reason + + +class TestCoerceResultScoreError: + """Tests for _coerce_result handling non-numeric score.""" + + def test_non_numeric_score_raises(self): + """Dict with non-numeric score raises TypeError.""" + with pytest.raises(TypeError, match="non-numeric 'score'"): + _coerce_result({"score": "high"}, "test_check") + + def test_none_score_raises(self): + with pytest.raises(TypeError, match="non-numeric 'score'"): + _coerce_result({"score": None}, "test_check") + + +class TestBareCheckViaEvaluateAgent: + """Test bare callable check functions through the public evaluate_agent API.""" + + async def test_bare_check_through_evaluate_agent(self): + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock( + return_value=AgentResponse(messages=[Message("assistant", ["The weather is sunny"])]) + ) + + is_long = keyword_check("weather") + + results = await evaluate_agent( + agent=mock_agent, + queries=["Q"], + evaluators=is_long, + ) + assert results[0].total == 1 + assert results[0].passed == 1 + + +class TestEvaluateAgentModuloWrapping: + """Test that expected_output stamps correctly with num_repetitions > 1 and multiple queries.""" + + async def test_modulo_stamps_correct_expected_output(self): + from unittest.mock import AsyncMock, MagicMock + + from agent_framework._evaluation import evaluate_agent + from agent_framework._types import AgentResponse + + mock_agent = MagicMock() + mock_agent.name = "test" + mock_agent.default_options = {} + mock_agent.run = AsyncMock(return_value=AgentResponse(messages=[Message("assistant", ["reply"])])) + + # Track which expected_output each item gets + seen_expected: list[str] = [] + + @evaluator + def capture_expected(response: str, expected_output: str) -> dict: + seen_expected.append(expected_output) + return {"passed": True, "reason": "ok"} + + await evaluate_agent( + agent=mock_agent, + queries=["Q1", "Q2", "Q3"], + expected_output=["A", "B", "C"], + evaluators=LocalEvaluator(capture_expected), + num_repetitions=2, + ) + # 3 queries × 2 reps = 6 items; modulo wrapping: A,B,C,A,B,C + assert seen_expected == ["A", "B", "C", "A", "B", "C"] + + +class TestEvaluateAgentQueriesWithoutAgent: + """Test error message when queries provided without agent.""" + + async def test_queries_without_agent_gives_clear_error(self): + from agent_framework._evaluation import evaluate_agent + + with pytest.raises(ValueError, match="Provide 'agent' when using 'queries'"): + await evaluate_agent( + queries=["hello"], + evaluators=LocalEvaluator(keyword_check("x")), + ) + + +# --------------------------------------------------------------------------- +# r5 review: all_passed with result_counts=None + sub_results +# --------------------------------------------------------------------------- + + +class TestAllPassedSubResults: + """Tests for EvalResults.all_passed with sub_results.""" + + def test_all_passed_ignores_own_counts_when_none(self): + """When result_counts is None (aggregate), all_passed delegates to sub_results.""" + from agent_framework._evaluation import EvalResults + + sub_pass = EvalResults( + provider="Local", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0, "errored": 0}, + ) + parent = EvalResults( + provider="Local", + eval_id="e0", + run_id="r0", + status="completed", + result_counts=None, + sub_results={"agent1": sub_pass}, + ) + assert parent.all_passed is True + + def test_all_passed_parent_fails_when_own_counts_fail(self): + """When parent has result_counts with failures, all_passed is False even if sub_results pass.""" + from agent_framework._evaluation import EvalResults + + sub_pass = EvalResults( + provider="Local", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0, "errored": 0}, + ) + parent = EvalResults( + provider="Local", + eval_id="e0", + run_id="r0", + status="completed", + result_counts={"passed": 1, "failed": 1, "errored": 0}, + sub_results={"agent1": sub_pass}, + ) + assert parent.all_passed is False + + +# --------------------------------------------------------------------------- +# r5 review: _build_overall_item with empty outputs +# --------------------------------------------------------------------------- + + +class TestBuildOverallItemEmpty: + """Test _build_overall_item returns None for empty workflow outputs.""" + + def test_returns_none_for_empty_outputs(self): + from unittest.mock import MagicMock + + from agent_framework._evaluation import _build_overall_item + + mock_result = MagicMock() + mock_result.get_outputs.return_value = [] + item = _build_overall_item("Hello", mock_result) + assert item is None diff --git a/python/packages/declarative/pyproject.toml b/python/packages/declarative/pyproject.toml index e14843fb6f..64bcd0a725 100644 --- a/python/packages/declarative/pyproject.toml +++ b/python/packages/declarative/pyproject.toml @@ -4,7 +4,7 @@ description = "Declarative specification support for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -22,7 +22,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "powerfx>=0.0.32,<0.0.35; python_version < '3.14'", "pyyaml>=6.0,<7.0", ] diff --git a/python/packages/devui/pyproject.toml b/python/packages/devui/pyproject.toml index f8441af3b0..1de35bd0cb 100644 --- a/python/packages/devui/pyproject.toml +++ b/python/packages/devui/pyproject.toml @@ -4,7 +4,7 @@ description = "Debug UI for Microsoft Agent Framework with OpenAI-compatible API authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://github.com/microsoft/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "openai>=1.99.0,<3", "opentelemetry-sdk>=1.39.0,<2", "fastapi>=0.115.0,<0.133.1", diff --git a/python/packages/durabletask/pyproject.toml b/python/packages/durabletask/pyproject.toml index 7d41b41000..f2fe9af94b 100644 --- a/python/packages/durabletask/pyproject.toml +++ b/python/packages/durabletask/pyproject.toml @@ -4,7 +4,7 @@ description = "Durable Task integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -22,7 +22,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "durabletask>=1.3.0,<2", "durabletask-azuremanaged>=1.3.0,<2", "python-dateutil>=2.8.0,<3", diff --git a/python/packages/foundry/agent_framework_foundry/__init__.py b/python/packages/foundry/agent_framework_foundry/__init__.py index 50c500ad4e..a67b5df801 100644 --- a/python/packages/foundry/agent_framework_foundry/__init__.py +++ b/python/packages/foundry/agent_framework_foundry/__init__.py @@ -4,6 +4,11 @@ import importlib.metadata from ._agent import FoundryAgent, RawFoundryAgent, RawFoundryAgentChatClient from ._chat_client import FoundryChatClient, FoundryChatOptions, RawFoundryChatClient +from ._foundry_evals import ( + FoundryEvals, + evaluate_foundry_target, + evaluate_traces, +) from ._memory_provider import FoundryMemoryProvider try: @@ -15,9 +20,12 @@ __all__ = [ "FoundryAgent", "FoundryChatClient", "FoundryChatOptions", + "FoundryEvals", "FoundryMemoryProvider", "RawFoundryAgent", "RawFoundryAgentChatClient", "RawFoundryChatClient", "__version__", + "evaluate_foundry_target", + "evaluate_traces", ] diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py new file mode 100644 index 0000000000..697941bbfc --- /dev/null +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -0,0 +1,891 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Microsoft Foundry Evals integration for Microsoft Agent Framework. + +Provides ``FoundryEvals``, an ``Evaluator`` implementation backed by Azure AI +Foundry's built-in evaluators. See docs/decisions/0018-foundry-evals-integration.md +for the design rationale. + +Example: + +.. code-block:: python + + from agent_framework import evaluate_agent + from agent_framework.foundry import FoundryEvals + + # Zero-config: reads FOUNDRY_PROJECT_ENDPOINT and FOUNDRY_MODEL from env + evals = FoundryEvals() + results = await evaluate_agent( + agent=my_agent, + queries=["What's the weather in Seattle?"], + evaluators=evals, + ) + results[0].raise_for_status() + print(results[0].report_url) +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from agent_framework._evaluation import ( + AgentEvalConverter, + ConversationSplit, + ConversationSplitter, + EvalItem, + EvalItemResult, + EvalResults, + EvalScoreResult, +) +from openai import AsyncOpenAI + +from ._chat_client import FoundryChatClient + +if TYPE_CHECKING: + from azure.ai.projects.aio import AIProjectClient + from openai.types.evals import RunRetrieveResponse + +logger = logging.getLogger(__name__) + +# Agent evaluators that accept query/response as conversation arrays. +# Maintained manually — check https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/evaluate-sdk +# for the latest evaluator list. These are the evaluators that need conversation-format input. +_AGENT_EVALUATORS: set[str] = { + "builtin.intent_resolution", + "builtin.task_adherence", + "builtin.task_completion", + "builtin.task_navigation_efficiency", + "builtin.tool_call_accuracy", + "builtin.tool_selection", + "builtin.tool_input_accuracy", + "builtin.tool_output_utilization", + "builtin.tool_call_success", +} + +# Evaluators that additionally require tool_definitions. +_TOOL_EVALUATORS: set[str] = { + "builtin.tool_call_accuracy", + "builtin.tool_selection", + "builtin.tool_input_accuracy", + "builtin.tool_output_utilization", + "builtin.tool_call_success", +} + +_BUILTIN_EVALUATORS: dict[str, str] = { + # Agent behavior + "intent_resolution": "builtin.intent_resolution", + "task_adherence": "builtin.task_adherence", + "task_completion": "builtin.task_completion", + "task_navigation_efficiency": "builtin.task_navigation_efficiency", + # Tool usage + "tool_call_accuracy": "builtin.tool_call_accuracy", + "tool_selection": "builtin.tool_selection", + "tool_input_accuracy": "builtin.tool_input_accuracy", + "tool_output_utilization": "builtin.tool_output_utilization", + "tool_call_success": "builtin.tool_call_success", + # Quality + "coherence": "builtin.coherence", + "fluency": "builtin.fluency", + "relevance": "builtin.relevance", + "groundedness": "builtin.groundedness", + "response_completeness": "builtin.response_completeness", + "similarity": "builtin.similarity", + # Safety + "violence": "builtin.violence", + "sexual": "builtin.sexual", + "self_harm": "builtin.self_harm", + "hate_unfairness": "builtin.hate_unfairness", +} + +# Default evaluator sets used when evaluators=None +_DEFAULT_EVALUATORS: list[str] = [ + "relevance", + "coherence", + "task_adherence", +] + +_DEFAULT_TOOL_EVALUATORS: list[str] = [ + "tool_call_accuracy", +] + +# Consistency between evaluator sets is enforced by tests in +# test_foundry_evals.py — see TestEvaluatorSetConsistency. + + +def _resolve_evaluator(name: str) -> str: + """Resolve a short evaluator name to its fully-qualified ``builtin.*`` form. + + Args: + name: Short name (e.g. ``"relevance"``) or fully-qualified name + (e.g. ``"builtin.relevance"``). + + Returns: + The fully-qualified evaluator name. + + Raises: + ValueError: If the name is not recognized. + """ + if name.startswith("builtin."): + # Already fully-qualified — pass through, but warn if not in our + # known list (may indicate a typo or a newly-added evaluator). + short = name.removeprefix("builtin.") + if short not in _BUILTIN_EVALUATORS: + logger.warning( + "Evaluator '%s' is not in the known built-in list. " + "If this is a new evaluator, consider updating _BUILTIN_EVALUATORS.", + name, + ) + return name + resolved = _BUILTIN_EVALUATORS.get(name) + if resolved is None: + raise ValueError(f"Unknown evaluator '{name}'. Available: {sorted(_BUILTIN_EVALUATORS)}") + return resolved + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _build_testing_criteria( + evaluators: Sequence[str], + model: str, + *, + include_data_mapping: bool = False, +) -> list[dict[str, Any]]: + """Build ``testing_criteria`` for ``evals.create()``. + + Args: + evaluators: Evaluator names. + model: Model deployment for the LLM judge. + include_data_mapping: Whether to include field-level data mapping + (required for the JSONL data source, not needed for response-based). + """ + criteria: list[dict[str, Any]] = [] + for name in evaluators: + qualified = _resolve_evaluator(name) + short = name if not name.startswith("builtin.") else name.split(".")[-1] + + # Structure dictated by the OpenAI evals API — see + # https://platform.openai.com/docs/api-reference/evals/create + entry: dict[str, Any] = { + "type": "azure_ai_evaluator", + "name": short, + "evaluator_name": qualified, + "initialization_parameters": {"deployment_name": model}, + } + + if include_data_mapping: + if qualified in _AGENT_EVALUATORS: + # Agent evaluators: query/response as conversation arrays. + # {{item.*}} are Mustache-style placeholders resolved by the + # evals API against fields in the JSONL data items. + mapping: dict[str, str] = { + "query": "{{item.query_messages}}", + "response": "{{item.response_messages}}", + } + else: + # Quality evaluators: query/response as strings + mapping = { + "query": "{{item.query}}", + "response": "{{item.response}}", + } + if qualified == "builtin.groundedness": + mapping["context"] = "{{item.context}}" + if qualified in _TOOL_EVALUATORS: + mapping["tool_definitions"] = "{{item.tool_definitions}}" + entry["data_mapping"] = mapping + + criteria.append(entry) + return criteria + + +def _build_item_schema(*, has_context: bool = False, has_tools: bool = False) -> dict[str, Any]: + """Build the ``item_schema`` for custom JSONL eval definitions.""" + properties: dict[str, Any] = { + "query": {"type": "string"}, + "response": {"type": "string"}, + "query_messages": {"type": "array"}, + "response_messages": {"type": "array"}, + } + if has_context: + properties["context"] = {"type": "string"} + if has_tools: + properties["tool_definitions"] = {"type": "array"} + return { + "type": "object", + "properties": properties, + "required": ["query", "response"], + } + + +def _resolve_default_evaluators( + evaluators: Sequence[str] | None, + items: Sequence[EvalItem | dict[str, Any]] | None = None, +) -> list[str]: + """Resolve evaluators, applying defaults when ``None``. + + Defaults to relevance + coherence + task_adherence. Automatically adds + tool_call_accuracy when items contain tools. + """ + if evaluators is not None: + return list(evaluators) + + result = list(_DEFAULT_EVALUATORS) + if items is not None: + has_tools = any((item.tools if isinstance(item, EvalItem) else item.get("tool_definitions")) for item in items) + if has_tools: + result.extend(_DEFAULT_TOOL_EVALUATORS) + return result + + +def _filter_tool_evaluators( + evaluators: list[str], + items: Sequence[EvalItem | dict[str, Any]], +) -> list[str]: + """Remove tool evaluators if no items have tool definitions.""" + has_tools = any((item.tools if isinstance(item, EvalItem) else item.get("tool_definitions")) for item in items) + if has_tools: + return evaluators + filtered = [e for e in evaluators if _resolve_evaluator(e) not in _TOOL_EVALUATORS] + if not filtered: + raise ValueError( + f"All requested evaluators {evaluators} require tool definitions, " + "but no items have tools. Either add tool definitions to your items " + "or choose evaluators that do not require tools." + ) + if len(filtered) < len(evaluators): + removed = [e for e in evaluators if _resolve_evaluator(e) in _TOOL_EVALUATORS] + logger.info("Removed tool evaluators %s (no items have tools)", removed) + return filtered + + +async def _poll_eval_run( + client: AsyncOpenAI, + eval_id: str, + run_id: str, + poll_interval: float = 5.0, + timeout: float = 180.0, + provider: str = "Microsoft Foundry", + *, + fetch_output_items: bool = True, +) -> EvalResults: + """Poll an eval run until completion or timeout.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while True: + run = await client.evals.runs.retrieve(run_id=run_id, eval_id=eval_id) + if run.status in ("completed", "failed", "canceled"): + error_msg = None + if run.status == "failed": + err = run.error + if err is not None: # pyright: ignore[reportUnnecessaryComparison] + error_msg = err if isinstance(err, str) else err.message or str(err) + + items: list[EvalItemResult] = [] + if fetch_output_items and run.status == "completed": + items = await _fetch_output_items(client, eval_id, run_id) + + return EvalResults( + provider=provider, + eval_id=eval_id, + run_id=run_id, + status=run.status, + result_counts=_extract_result_counts(run), + report_url=run.report_url, + error=error_msg, + per_evaluator=_extract_per_evaluator(run), + items=items, + ) + remaining = deadline - loop.time() + if remaining <= 0: + return EvalResults(provider=provider, eval_id=eval_id, run_id=run_id, status="timeout") + logger.debug("Eval run %s status: %s (%.0fs remaining)", run_id, run.status, remaining) + await asyncio.sleep(min(poll_interval, remaining)) + + +def _extract_result_counts(run: RunRetrieveResponse) -> dict[str, int] | None: + """Extract result_counts from an eval run as a plain dict.""" + counts = run.result_counts + if counts is None: # pyright: ignore[reportUnnecessaryComparison] + return None + return { + "errored": counts.errored, + "failed": counts.failed, + "passed": counts.passed, + "total": counts.total, + } + + +def _extract_per_evaluator(run: RunRetrieveResponse) -> dict[str, dict[str, int]]: + """Extract per-evaluator result breakdowns from an eval run.""" + per_eval: dict[str, dict[str, int]] = {} + for item in run.per_testing_criteria_results or []: + name = item.testing_criteria + if name: + per_eval[name] = {"passed": item.passed, "failed": item.failed} + return per_eval + + +async def _fetch_output_items( + client: AsyncOpenAI, + eval_id: str, + run_id: str, +) -> list[EvalItemResult]: + """Fetch per-item results from the output_items API. + + Converts the provider-specific ``OutputItemListResponse`` objects into + provider-agnostic ``EvalItemResult`` instances with per-evaluator scores, + error categorization, and token usage. Uses async pagination to handle + eval runs with more items than a single page. + """ + items: list[EvalItemResult] = [] + try: + output_items_page = await client.evals.runs.output_items.list( + run_id=run_id, + eval_id=eval_id, + ) + + async for oi in output_items_page: + # Extract per-evaluator scores + scores: list[EvalScoreResult] = [] + for r in oi.results or []: + scores.append( + EvalScoreResult( + name=r.name, + score=r.score, + passed=r.passed, + sample=r.sample, + ) + ) + + # Extract error info from sample + error_code: str | None = None + error_message: str | None = None + token_usage: dict[str, int] | None = None + input_text: str | None = None + output_text: str | None = None + response_id: str | None = None + + sample = oi.sample + if sample is not None: # pyright: ignore[reportUnnecessaryComparison] + err = sample.error + if err is not None and (err.code or err.message): # pyright: ignore[reportUnnecessaryComparison] + error_code = err.code or None + error_message = err.message or None + + usage = sample.usage + if usage is not None and usage.total_tokens: # pyright: ignore[reportUnnecessaryComparison] + token_usage = { + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + "cached_tokens": usage.cached_tokens, + } + + # Extract input/output text + if sample.input: + parts = [si.content for si in sample.input if si.role == "user"] + if parts: + input_text = " ".join(parts) + + if sample.output: + parts = [so.content or "" for so in sample.output if so.role == "assistant"] + if parts: + output_text = " ".join(parts) + + # Extract response_id from datasource_item + ds_item = oi.datasource_item + if ds_item: + resp_id_val = ds_item.get("resp_id") or ds_item.get("response_id") + response_id = str(resp_id_val) if resp_id_val else None + + items.append( + EvalItemResult( + item_id=oi.id, + status=oi.status, + scores=scores, + error_code=error_code, + error_message=error_message, + response_id=response_id, + input_text=input_text, + output_text=output_text, + token_usage=token_usage, + ) + ) + except (AttributeError, KeyError, TypeError): + logger.warning("Could not fetch output_items for run %s", run_id, exc_info=True) + + return items + + +def _resolve_openai_client( + client: FoundryChatClient | AsyncOpenAI | None = None, + project_client: AIProjectClient | None = None, +) -> AsyncOpenAI: + """Resolve an AsyncOpenAI client from a FoundryChatClient, raw client, or project_client.""" + if client is not None: + if isinstance(client, FoundryChatClient): + return client.client + return client + if project_client is not None: + oai = project_client.get_openai_client() + if oai is None: # pyright: ignore[reportUnnecessaryComparison] + raise ValueError("project_client.get_openai_client() returned None. Check project configuration.") + if not isinstance(oai, AsyncOpenAI): + raise TypeError( + "project_client.get_openai_client() returned a sync client. " + "FoundryEvals requires an async AIProjectClient (from azure.ai.projects.aio)." + ) + return oai + raise ValueError("Provide either 'client' or 'project_client'.") + + +async def _evaluate_via_responses_impl( + *, + client: AsyncOpenAI, + response_ids: Sequence[str], + evaluators: list[str], + model: str, + eval_name: str, + poll_interval: float, + timeout: float, + provider: str = "foundry", +) -> EvalResults: + """Evaluate using Foundry's Responses API retrieval path. + + Module-level helper used by both ``FoundryEvals`` and ``evaluate_traces``. + """ + eval_obj = await client.evals.create( + name=eval_name, + data_source_config={"type": "azure_ai_source", "scenario": "responses"}, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + testing_criteria=_build_testing_criteria(evaluators, model), # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + data_source = { + "type": "azure_ai_responses", + "item_generation_params": { + "type": "response_retrieval", + "data_mapping": {"response_id": "{{item.resp_id}}"}, + "source": { + "type": "file_content", + "content": [{"item": {"resp_id": rid}} for rid in response_ids], + }, + }, + } + + run = await client.evals.runs.create( + eval_id=eval_obj.id, + name=f"{eval_name} Run", + data_source=data_source, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + return await _poll_eval_run(client, eval_obj.id, run.id, poll_interval, timeout, provider=provider) + + +# --------------------------------------------------------------------------- +# FoundryEvals — Evaluator implementation for Microsoft Foundry +# --------------------------------------------------------------------------- + + +class FoundryEvals: + """Evaluation provider backed by Microsoft Foundry. + + Implements the ``Evaluator`` protocol so it can be passed to the + provider-agnostic ``evaluate_agent()`` and + ``evaluate_workflow()`` functions from ``agent_framework``. + + Also provides constants for built-in evaluator names for IDE + autocomplete and typo prevention: + + .. code-block:: python + + from agent_framework.foundry import FoundryEvals + + evaluators = [FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY] + + Examples: + Basic usage: + + .. code-block:: python + + from agent_framework import evaluate_agent + from agent_framework.foundry import FoundryEvals, FoundryChatClient + + chat_client = FoundryChatClient(model="gpt-4o") + evals = FoundryEvals(client=chat_client) + results = await evaluate_agent(agent=agent, queries=queries, evaluators=evals) + + Zero-config with environment variables (``FOUNDRY_PROJECT_ENDPOINT`` + and ``FOUNDRY_MODEL``): + + .. code-block:: python + + evals = FoundryEvals() # reads env vars via FoundryChatClient + + **Evaluator selection:** + + By default, runs ``relevance``, ``coherence``, and ``task_adherence``. + Automatically adds ``tool_call_accuracy`` when items contain tool + definitions. Override with ``evaluators=``. + + .. note:: + + The ``builtin.*`` evaluators are accessed through the OpenAI Evals + API (``client.evals.create`` / ``client.evals.runs.create``). Any + ``AsyncOpenAI`` client pointing at a Foundry endpoint can run them. + + Args: + client: A ``FoundryChatClient`` instance. The ``builtin.*`` + evaluators are a Foundry feature and require a Foundry endpoint. + When omitted (and *project_client* is also omitted), a + ``FoundryChatClient`` is auto-created from ``FOUNDRY_PROJECT_ENDPOINT`` + and ``FOUNDRY_MODEL`` environment variables. + project_client: An async ``AIProjectClient`` instance + (from ``azure.ai.projects.aio``). Provide this or *client*. + model: Model deployment name for the evaluator LLM judge. + Resolved from ``client.model`` when omitted. + evaluators: Evaluator names (e.g. ``["relevance", "tool_call_accuracy"]``). + When ``None`` (default), uses smart defaults based on item data. + conversation_split: How to split multi-turn conversations into + query/response halves. Defaults to ``LAST_TURN``. Pass a + ``ConversationSplit`` enum value or a custom callable — see + ``ConversationSplitter``. + poll_interval: Seconds between status polls (default 5.0). + timeout: Maximum seconds to wait for completion (default 180.0). + eval_name: Display name for the eval definition created in Foundry. + Defaults to ``"agent-framework-eval"``. The name is visible in + the Foundry portal; it does not affect evaluation behavior. + """ + + # --------------------------------------------------------------------------- + # Built-in evaluator name constants + # --------------------------------------------------------------------------- + + # Agent behavior + INTENT_RESOLUTION: str = "intent_resolution" + TASK_ADHERENCE: str = "task_adherence" + TASK_COMPLETION: str = "task_completion" + TASK_NAVIGATION_EFFICIENCY: str = "task_navigation_efficiency" + + # Tool usage + TOOL_CALL_ACCURACY: str = "tool_call_accuracy" + TOOL_SELECTION: str = "tool_selection" + TOOL_INPUT_ACCURACY: str = "tool_input_accuracy" + TOOL_OUTPUT_UTILIZATION: str = "tool_output_utilization" + TOOL_CALL_SUCCESS: str = "tool_call_success" + + # Quality + COHERENCE: str = "coherence" + FLUENCY: str = "fluency" + RELEVANCE: str = "relevance" + GROUNDEDNESS: str = "groundedness" + RESPONSE_COMPLETENESS: str = "response_completeness" + SIMILARITY: str = "similarity" + + # Safety + VIOLENCE: str = "violence" + SEXUAL: str = "sexual" + SELF_HARM: str = "self_harm" + HATE_UNFAIRNESS: str = "hate_unfairness" + + def __init__( + self, + *, + client: FoundryChatClient | None = None, + project_client: AIProjectClient | None = None, + model: str | None = None, + evaluators: Sequence[str] | None = None, + conversation_split: ConversationSplitter = ConversationSplit.LAST_TURN, + poll_interval: float = 5.0, + timeout: float = 180.0, + ): + self.name = "Microsoft Foundry" + + # Auto-create a FoundryChatClient from env vars when no client is provided + if client is None and project_client is None: + client = FoundryChatClient(model=model or "gpt-4o") + + self._client = _resolve_openai_client(client, project_client) + # Resolve model: explicit param > client.model > error + resolved_model = model or (client.model if client is not None else None) + if not resolved_model: + raise ValueError( + "Model is required. Pass model= explicitly or use a FoundryChatClient that has a model configured." + ) + self._model = resolved_model + self._evaluators = list(evaluators) if evaluators is not None else None + self._conversation_split = conversation_split + self._poll_interval = poll_interval + self._timeout = timeout + + async def evaluate( + self, + items: Sequence[EvalItem], + *, + eval_name: str = "Agent Framework Eval", + ) -> EvalResults: + """Evaluate items using Foundry evaluators. + + Implements the ``Evaluator`` protocol. Automatically resolves default + evaluators and filters tool evaluators for items without tool definitions. + + Args: + items: Eval data items from ``AgentEvalConverter.to_eval_item()``. + eval_name: Display name for the evaluation run. + + Returns: + ``EvalResults`` with status, counts, and portal link. + """ + # Resolve evaluators with auto-detection + resolved = _resolve_default_evaluators(self._evaluators, items=items) + # Filter tool evaluators if items don't have tools + resolved = _filter_tool_evaluators(resolved, items) + + # Standard JSONL dataset path + return await self._evaluate_via_dataset(items, resolved, eval_name) + + # -- Internal evaluation paths -- + + async def _evaluate_via_dataset( + self, + items: Sequence[EvalItem], + evaluators: list[str], + eval_name: str, + ) -> EvalResults: + """Evaluate using JSONL dataset upload path.""" + dicts: list[dict[str, Any]] = [] + for item in items: + # Build JSONL dict directly from split_messages + converter + # to avoid splitting the conversation twice. + effective_split = item.split_strategy or self._conversation_split + query_msgs, response_msgs = item.split_messages(effective_split) + + query_text = " ".join(m.text for m in query_msgs if m.role == "user" and m.text).strip() + response_text = " ".join(m.text for m in response_msgs if m.role == "assistant" and m.text).strip() + + d: dict[str, Any] = { + "query": query_text, + "response": response_text, + "query_messages": AgentEvalConverter.convert_messages(query_msgs), + "response_messages": AgentEvalConverter.convert_messages(response_msgs), + } + if item.tools: + d["tool_definitions"] = [ + {"name": t.name, "description": t.description, "parameters": t.parameters()} for t in item.tools + ] + if item.context: + d["context"] = item.context + dicts.append(d) + + has_context = any("context" in d for d in dicts) + has_tools = any("tool_definitions" in d for d in dicts) + + eval_obj = await self._client.evals.create( + name=eval_name, + data_source_config={ # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + "type": "custom", + "item_schema": _build_item_schema(has_context=has_context, has_tools=has_tools), + "include_sample_schema": True, + }, + testing_criteria=_build_testing_criteria( # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + evaluators, + self._model, + include_data_mapping=True, + ), + ) + + data_source = { + "type": "jsonl", + "source": { + "type": "file_content", + "content": [{"item": d} for d in dicts], + }, + } + + run = await self._client.evals.runs.create( + eval_id=eval_obj.id, + name=f"{eval_name} Run", + data_source=data_source, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + return await _poll_eval_run( + self._client, + eval_obj.id, + run.id, + self._poll_interval, + self._timeout, + provider=self.name, + ) + + +# --------------------------------------------------------------------------- +# Foundry-specific functions (not part of the Evaluator protocol) +# --------------------------------------------------------------------------- + + +async def evaluate_traces( + *, + evaluators: Sequence[str] | None = None, + client: FoundryChatClient | None = None, + project_client: AIProjectClient | None = None, + model: str, + response_ids: Sequence[str] | None = None, + trace_ids: Sequence[str] | None = None, + agent_id: str | None = None, + lookback_hours: int = 24, + eval_name: str = "Agent Framework Trace Eval", + poll_interval: float = 5.0, + timeout: float = 180.0, +) -> EvalResults: + """Evaluate agent behavior from OTel traces or response IDs. + + Foundry-specific function — works with any agent that emits OTel traces + to App Insights. Provide *response_ids* for specific responses, + *trace_ids* for specific traces, or *agent_id* with *lookback_hours* + to evaluate recent activity. + + Args: + evaluators: Evaluator names (e.g. ``[FoundryEvals.RELEVANCE]``). + Defaults to relevance, coherence, and task_adherence. + client: A ``FoundryChatClient`` instance. Provide this or *project_client*. + project_client: An ``AIProjectClient`` instance. + model: Model deployment name for the evaluator LLM judge. + response_ids: Evaluate specific Responses API responses. + trace_ids: Evaluate specific OTel trace IDs from App Insights. + agent_id: Filter traces by agent ID (used with *lookback_hours*). + lookback_hours: Hours of trace history to evaluate (default 24). + eval_name: Display name for the evaluation. + poll_interval: Seconds between status polls. + timeout: Maximum seconds to wait for completion. + + Returns: + ``EvalResults`` with status, result counts, and portal link. + + Example: + + .. code-block:: python + + results = await evaluate_traces( + response_ids=[response.response_id], + evaluators=[FoundryEvals.RELEVANCE], + client=chat_client, + model="gpt-4o", + ) + """ + oai_client = _resolve_openai_client(client, project_client) + resolved_evaluators = _resolve_default_evaluators(evaluators) + + if response_ids: + return await _evaluate_via_responses_impl( + client=oai_client, + response_ids=response_ids, + evaluators=resolved_evaluators, + model=model, + eval_name=eval_name, + poll_interval=poll_interval, + timeout=timeout, + ) + + if not trace_ids and not agent_id: + raise ValueError("Provide at least one of: response_ids, trace_ids, or agent_id") + + trace_source: dict[str, Any] = { + "type": "azure_ai_traces", + "lookback_hours": lookback_hours, + } + if trace_ids: + trace_source["trace_ids"] = list(trace_ids) + if agent_id: + trace_source["agent_id"] = agent_id + + eval_obj = await oai_client.evals.create( + name=eval_name, + data_source_config={"type": "azure_ai_source", "scenario": "traces"}, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + testing_criteria=_build_testing_criteria(resolved_evaluators, model), # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + run = await oai_client.evals.runs.create( + eval_id=eval_obj.id, + name=f"{eval_name} Run", + data_source=trace_source, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + return await _poll_eval_run(oai_client, eval_obj.id, run.id, poll_interval, timeout) + + +async def evaluate_foundry_target( + *, + target: dict[str, Any], + test_queries: Sequence[str], + evaluators: Sequence[str] | None = None, + client: FoundryChatClient | None = None, + project_client: AIProjectClient | None = None, + model: str, + eval_name: str = "Agent Framework Target Eval", + poll_interval: float = 5.0, + timeout: float = 180.0, +) -> EvalResults: + """Evaluate a Foundry-registered agent or model deployment. + + Foundry invokes the target, captures the output, and evaluates it. Use + this for scheduled evals, red teaming, and CI/CD quality gates. + + Args: + target: Target configuration dict. + test_queries: Queries for Foundry to send to the target. + evaluators: Evaluator names. + client: A ``FoundryChatClient`` instance. Provide this or *project_client*. + project_client: An ``AIProjectClient`` instance. + model: Model deployment name for the evaluator LLM judge. + eval_name: Display name for the evaluation. + poll_interval: Seconds between status polls. + timeout: Maximum seconds to wait for completion. + + Returns: + ``EvalResults`` with status, result counts, and portal link. + + Example: + + .. code-block:: python + + results = await evaluate_foundry_target( + target={"type": "azure_ai_agent", "name": "my-agent"}, + test_queries=["Book a flight to Paris"], + client=chat_client, + model="gpt-4o", + ) + """ + if "type" not in target: + raise ValueError("target dict must include a 'type' key (e.g., 'azure_ai_agent').") + oai_client = _resolve_openai_client(client, project_client) + resolved_evaluators = _resolve_default_evaluators(evaluators) + + eval_obj = await oai_client.evals.create( + name=eval_name, + data_source_config={ # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + "type": "azure_ai_source", + "scenario": "target_completions", + }, + testing_criteria=_build_testing_criteria(resolved_evaluators, model), # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + data_source: dict[str, Any] = { + "type": "azure_ai_target_completions", + "target": target, + "source": { + "type": "file_content", + "content": [{"item": {"query": q}} for q in test_queries], + }, + } + + run = await oai_client.evals.runs.create( + eval_id=eval_obj.id, + name=f"{eval_name} Run", + data_source=data_source, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + ) + + return await _poll_eval_run(oai_client, eval_obj.id, run.id, poll_interval, timeout) diff --git a/python/packages/foundry/pyproject.toml b/python/packages/foundry/pyproject.toml index 2cc81a5e43..d07b5087fd 100644 --- a/python/packages/foundry/pyproject.toml +++ b/python/packages/foundry/pyproject.toml @@ -4,7 +4,7 @@ description = "Cloud Azure AI Foundry integration for Microsoft Agent Framework. authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0rc5" +version = "1.0.0rc6" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,8 +23,8 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", - "agent-framework-openai>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", + "agent-framework-openai>=1.0.0rc6", "azure-ai-projects>=2.0.0,<3.0", ] diff --git a/python/packages/foundry/tests/test_foundry_evals.py b/python/packages/foundry/tests/test_foundry_evals.py new file mode 100644 index 0000000000..cef890c7af --- /dev/null +++ b/python/packages/foundry/tests/test_foundry_evals.py @@ -0,0 +1,2591 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for the AgentEvalConverter, FoundryEvals, and eval helper functions.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from agent_framework import AgentExecutorResponse, AgentResponse, Content, FunctionTool, Message, WorkflowEvent +from agent_framework._evaluation import ( + AgentEvalConverter, + ConversationSplit, + EvalItem, + EvalNotPassedError, + EvalResults, + _extract_agent_eval_data, + _extract_overall_query, + evaluate_agent, + evaluate_workflow, +) +from agent_framework._workflows._workflow import WorkflowRunResult +from openai import AsyncOpenAI + +from agent_framework_foundry._foundry_evals import ( + FoundryEvals, + _build_item_schema, + _build_testing_criteria, + _extract_per_evaluator, + _extract_result_counts, + _filter_tool_evaluators, + _resolve_default_evaluators, + _resolve_evaluator, + _resolve_openai_client, +) + + +class _AsyncPage: + """Async-iterable mock for OpenAI SDK pagination pages.""" + + def __init__(self, items: list[Any]) -> None: + self._items = items + + def __aiter__(self) -> _AsyncPage: + self._iter = iter(self._items) + return self + + async def __anext__(self) -> Any: + try: + return next(self._iter) + except StopIteration: + raise StopAsyncIteration from None + + +def _make_tool(name: str) -> MagicMock: + """Create a mock FunctionTool for use in tests.""" + t = MagicMock() + t.name = name + t.description = f"{name} tool" + t.parameters = MagicMock(return_value={"type": "object"}) + return t + + +@dataclass +class _MockResultCounts: + """Mock matching the OpenAI SDK ResultCounts Pydantic model shape.""" + + passed: int = 0 + failed: int = 0 + errored: int = 0 + total: int = 0 + + +def _rc(passed: int = 0, failed: int = 0, errored: int = 0) -> _MockResultCounts: + """Shorthand to create a ResultCounts-compatible mock.""" + return _MockResultCounts(passed=passed, failed=failed, errored=errored, total=passed + failed + errored) + + +# --------------------------------------------------------------------------- +# _resolve_evaluator +# --------------------------------------------------------------------------- + + +class TestResolveEvaluator: + def test_short_name(self) -> None: + assert _resolve_evaluator("relevance") == "builtin.relevance" + assert _resolve_evaluator("tool_call_accuracy") == "builtin.tool_call_accuracy" + assert _resolve_evaluator("violence") == "builtin.violence" + + def test_already_qualified(self) -> None: + assert _resolve_evaluator("builtin.relevance") == "builtin.relevance" + assert _resolve_evaluator("builtin.custom") == "builtin.custom" + + def test_unknown_raises(self) -> None: + with pytest.raises(ValueError, match="Unknown evaluator 'bogus'"): + _resolve_evaluator("bogus") + + +# --------------------------------------------------------------------------- +# AgentEvalConverter.convert_message +# --------------------------------------------------------------------------- + + +class TestConvertMessage: + def test_user_text_message(self) -> None: + msg = Message("user", ["Hello, world!"]) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0] == {"role": "user", "content": [{"type": "text", "text": "Hello, world!"}]} + + def test_system_message(self) -> None: + msg = Message("system", ["You are helpful."]) + result = AgentEvalConverter.convert_message(msg) + assert result[0] == {"role": "system", "content": [{"type": "text", "text": "You are helpful."}]} + + def test_assistant_text_message(self) -> None: + msg = Message("assistant", ["Here is the answer."]) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["role"] == "assistant" + assert result[0]["content"] == [{"type": "text", "text": "Here is the answer."}] + assert len(result[0]["content"]) == 1 + + def test_assistant_with_tool_call(self) -> None: + msg = Message( + "assistant", + [ + Content.from_function_call( + call_id="call_1", + name="get_weather", + arguments=json.dumps({"location": "Seattle"}), + ), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["role"] == "assistant" + tc = result[0]["content"][0] + assert tc["type"] == "tool_call" + assert tc["tool_call_id"] == "call_1" + assert tc["name"] == "get_weather" + assert tc["arguments"] == {"location": "Seattle"} + + def test_assistant_text_and_tool_call(self) -> None: + msg = Message( + "assistant", + [ + Content.from_text("Let me check that."), + Content.from_function_call( + call_id="call_2", + name="search", + arguments={"query": "flights"}, + ), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} + tc = result[0]["content"][1] + assert tc["type"] == "tool_call" + assert tc["arguments"] == {"query": "flights"} + + def test_tool_result_message(self) -> None: + msg = Message( + "tool", + [ + Content.from_function_result( + call_id="call_1", + result="72°F, sunny", + ), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["role"] == "tool" + assert result[0]["tool_call_id"] == "call_1" + assert result[0]["content"] == [{"type": "tool_result", "tool_result": "72°F, sunny"}] + + def test_multiple_tool_results(self) -> None: + msg = Message( + "tool", + [ + Content.from_function_result(call_id="call_1", result="r1"), + Content.from_function_result(call_id="call_2", result="r2"), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 2 + assert result[0]["tool_call_id"] == "call_1" + assert result[1]["tool_call_id"] == "call_2" + + def test_non_string_result_kept_as_object(self) -> None: + msg = Message( + "tool", + [ + Content.from_function_result( + call_id="call_1", + result={"temp": 72, "unit": "F"}, + ), + ], + ) + result = AgentEvalConverter.convert_message(msg) + tr = result[0]["content"][0] + assert tr["type"] == "tool_result" + assert tr["tool_result"] == {"temp": 72, "unit": "F"} + + def test_empty_message(self) -> None: + msg = Message("user", []) + result = AgentEvalConverter.convert_message(msg) + assert result[0] == {"role": "user", "content": [{"type": "text", "text": ""}]} + + def test_user_image_from_data(self) -> None: + """Image created via Content.from_data() emits input_image.""" + img = Content.from_data(data=b"\x89PNG\r\n\x1a\n", media_type="image/png") + msg = Message("user", [img]) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert result[0]["role"] == "user" + part = result[0]["content"][0] + assert part["type"] == "input_image" + assert part["image_url"].startswith("data:image/png;base64,") + assert part["detail"] == "auto" + + def test_user_image_from_uri(self) -> None: + """Image created via Content.from_uri() with an external URL.""" + img = Content.from_uri("https://example.com/photo.jpg", media_type="image/jpeg") + msg = Message("user", [img]) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + part = result[0]["content"][0] + assert part["type"] == "input_image" + assert part["image_url"] == "https://example.com/photo.jpg" + assert part["detail"] == "auto" + + def test_user_image_uri_without_media_type(self) -> None: + """URI content without media_type still emits input_image (no detail key).""" + img = Content("uri", uri="https://example.com/pic.png") + msg = Message("user", [img]) + result = AgentEvalConverter.convert_message(msg) + part = result[0]["content"][0] + assert part["type"] == "input_image" + assert part["image_url"] == "https://example.com/pic.png" + assert "detail" not in part + + def test_mixed_text_and_image(self) -> None: + """Message with text + image produces both content parts.""" + msg = Message( + "user", + [ + Content.from_text("What's in this image?"), + Content.from_uri("https://example.com/cat.jpg", media_type="image/jpeg"), + ], + ) + result = AgentEvalConverter.convert_message(msg) + assert len(result) == 1 + assert len(result[0]["content"]) == 2 + assert result[0]["content"][0] == {"type": "text", "text": "What's in this image?"} + assert result[0]["content"][1]["type"] == "input_image" + assert result[0]["content"][1]["image_url"] == "https://example.com/cat.jpg" + + +# --------------------------------------------------------------------------- +# AgentEvalConverter.convert_messages +# --------------------------------------------------------------------------- + + +class TestConvertMessages: + def test_full_conversation(self) -> None: + messages = [ + Message("user", ["What's the weather?"]), + Message( + "assistant", + [Content.from_function_call(call_id="c1", name="get_weather", arguments='{"loc": "SEA"}')], + ), + Message("tool", [Content.from_function_result(call_id="c1", result="Sunny")]), + Message("assistant", ["It's sunny in Seattle!"]), + ] + result = AgentEvalConverter.convert_messages(messages) + assert len(result) == 4 + assert result[0]["role"] == "user" + assert result[1]["role"] == "assistant" + assert result[1]["content"][0]["type"] == "tool_call" + assert result[1]["content"][0]["name"] == "get_weather" + assert result[2]["role"] == "tool" + assert result[2]["content"][0]["type"] == "tool_result" + assert result[3]["role"] == "assistant" + assert result[3]["content"] == [{"type": "text", "text": "It's sunny in Seattle!"}] + + def test_multimodal_conversation_preserves_images(self) -> None: + """Full conversation with image content flows through convert_messages.""" + messages = [ + Message( + "user", + [ + Content.from_text("Describe this image"), + Content.from_uri("https://example.com/photo.jpg", media_type="image/jpeg"), + ], + ), + Message("assistant", ["This is a photo of a sunset over the ocean."]), + ] + result = AgentEvalConverter.convert_messages(messages) + assert len(result) == 2 + # User message has text + image + user_content = result[0]["content"] + assert len(user_content) == 2 + assert user_content[0] == {"type": "text", "text": "Describe this image"} + assert user_content[1]["type"] == "input_image" + assert user_content[1]["image_url"] == "https://example.com/photo.jpg" + # Assistant response is text + assert result[1]["content"] == [{"type": "text", "text": "This is a photo of a sunset over the ocean."}] + + +# --------------------------------------------------------------------------- +# AgentEvalConverter.extract_tools +# --------------------------------------------------------------------------- + + +class TestExtractTools: + def test_extracts_function_tools(self) -> None: + tool = FunctionTool( + name="get_weather", + description="Get weather for a location", + func=lambda location: f"Sunny in {location}", + ) + agent = MagicMock() + agent.default_options = {"tools": [tool]} + + result = AgentEvalConverter.extract_tools(agent) + assert len(result) == 1 + assert result[0]["name"] == "get_weather" + assert result[0]["description"] == "Get weather for a location" + assert "parameters" in result[0] + + def test_skips_non_function_tools(self) -> None: + agent = MagicMock() + agent.default_options = {"tools": [{"type": "web_search"}, "some_string"]} + + result = AgentEvalConverter.extract_tools(agent) + assert len(result) == 0 + + def test_no_tools(self) -> None: + agent = MagicMock() + agent.default_options = {} + assert AgentEvalConverter.extract_tools(agent) == [] + + def test_no_default_options(self) -> None: + agent = MagicMock(spec=[]) # No attributes + assert AgentEvalConverter.extract_tools(agent) == [] + + +# --------------------------------------------------------------------------- +# AgentEvalConverter.to_eval_item (now returns EvalItem) +# --------------------------------------------------------------------------- + + +class TestToEvalItem: + def test_string_query(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["The weather is sunny."])]) + item = AgentEvalConverter.to_eval_item(query="What's the weather?", response=response) + + assert isinstance(item, EvalItem) + assert item.query == "What's the weather?" + assert item.response == "The weather is sunny." + assert len(item.conversation) == 2 + assert item.conversation[0].role == "user" + assert item.conversation[1].role == "assistant" + + def test_message_query(self) -> None: + input_msgs = [ + Message("system", ["Be helpful."]), + Message("user", ["Hello"]), + ] + response = AgentResponse(messages=[Message("assistant", ["Hi there!"])]) + item = AgentEvalConverter.to_eval_item(query=input_msgs, response=response) + + assert item.query == "Hello" # Only user messages + assert len(item.conversation) == 3 # system + user + assistant + + def test_with_context(self) -> None: + response = AgentResponse(messages=[Message("assistant", ["Answer."])]) + item = AgentEvalConverter.to_eval_item( + query="Question?", + response=response, + context="Some reference document.", + ) + assert item.context == "Some reference document." + + def test_with_explicit_tools(self) -> None: + tool = FunctionTool( + name="search", + description="Search the web", + func=lambda q: f"Results for {q}", + ) + response = AgentResponse(messages=[Message("assistant", ["Found it."])]) + item = AgentEvalConverter.to_eval_item( + query="Find info", + response=response, + tools=[tool], + ) + assert item.tools is not None + assert len(item.tools) == 1 + assert item.tools[0].name == "search" + + def test_with_agent_tools(self) -> None: + tool = FunctionTool(name="calc", description="Calculate", func=lambda x: str(x)) + agent = MagicMock() + agent.default_options = {"tools": [tool]} + + response = AgentResponse(messages=[Message("assistant", ["42"])]) + item = AgentEvalConverter.to_eval_item( + query="What is 6*7?", + response=response, + agent=agent, + ) + assert item.tools is not None + assert item.tools[0].name == "calc" + + def test_explicit_tools_override_agent(self) -> None: + agent_tool = FunctionTool(name="agent_tool", description="from agent", func=lambda: "") + explicit_tool = FunctionTool(name="explicit_tool", description="explicit", func=lambda: "") + + agent = MagicMock() + agent.default_options = {"tools": [agent_tool]} + + response = AgentResponse(messages=[Message("assistant", ["Done"])]) + item = AgentEvalConverter.to_eval_item( + query="Test", + response=response, + agent=agent, + tools=[explicit_tool], + ) + assert item.tools is not None + assert len(item.tools) == 1 + assert item.tools[0].name == "explicit_tool" + + def test_split_messages_format(self) -> None: + """split_messages() should split conversation at last user message.""" + response = AgentResponse(messages=[Message("assistant", ["Answer"])]) + item = AgentEvalConverter.to_eval_item( + query="Q", + response=response, + tools=[FunctionTool(name="t", description="d", func=lambda: "")], + ) + query_msgs, response_msgs = item.split_messages() + # Single-turn: query has just the user msg, response has the assistant msg + assert len(query_msgs) == 1 + assert query_msgs[0].role == "user" + assert len(response_msgs) == 1 + assert response_msgs[0].role == "assistant" + # Tools preserved on item + assert item.tools is not None + assert len(item.tools) == 1 + assert item.tools[0].name == "t" + + def test_split_messages_multiturn_preserves_interleaving(self) -> None: + """Multi-turn split_messages() splits at last user message, preserving interleaving.""" + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's sunny in Seattle."]), + Message("user", ["And tomorrow?"]), + Message("assistant", [Content(type="function_call", name="get_forecast")]), + Message("tool", [Content(type="function_result", result="Rain expected")]), + Message("assistant", ["Rain is expected tomorrow."]), + ] + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages() + # query_messages: everything up to and including the last user message + assert len(query_msgs) == 3 # user, assistant, user + assert query_msgs[0].role == "user" + assert query_msgs[1].role == "assistant" # interleaved! + assert query_msgs[2].role == "user" + # response_messages: everything after the last user message + assert len(response_msgs) == 3 # assistant(tool_call), tool, assistant + assert response_msgs[0].role == "assistant" + assert response_msgs[1].role == "tool" + assert response_msgs[2].role == "assistant" + + def test_split_messages_full_split(self) -> None: + """ConversationSplit.FULL splits after the first user message.""" + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's 62°F in Seattle."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["Rain is expected tomorrow."]), + ] + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=ConversationSplit.FULL) + # query_messages: just the first user message + assert len(query_msgs) == 1 + assert query_msgs[0].role == "user" + assert query_msgs[0].text == "What's the weather?" + # response_messages: everything after the first user message + assert len(response_msgs) == 3 + assert response_msgs[0].role == "assistant" + assert response_msgs[1].role == "user" + assert response_msgs[2].role == "assistant" + + def test_split_messages_full_split_with_system(self) -> None: + """FULL split includes system messages before the first user message in query.""" + conversation = [ + Message("system", ["You are a weather assistant."]), + Message("user", ["What's the weather?"]), + Message("assistant", ["It's sunny."]), + ] + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=ConversationSplit.FULL) + # query includes system + first user + assert len(query_msgs) == 2 + assert query_msgs[0].role == "system" + assert query_msgs[1].role == "user" + assert len(response_msgs) == 1 + + def test_split_messages_full_split_with_tools(self) -> None: + """FULL split puts all tool interactions in response_messages.""" + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", [Content(type="function_call", name="get_weather")]), + Message("tool", [Content(type="function_result", result="62°F")]), + Message("assistant", ["It's 62°F."]), + Message("user", ["Thanks!"]), + Message("assistant", ["You're welcome!"]), + ] + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=ConversationSplit.FULL) + assert len(query_msgs) == 1 + assert len(response_msgs) == 5 + + def test_split_messages_last_turn_is_default(self) -> None: + """Default split_messages() uses LAST_TURN split.""" + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi there"]), + Message("user", ["Bye"]), + Message("assistant", ["Goodbye"]), + ] + item = EvalItem(conversation=conversation) + q_default, r_default = item.split_messages() + q_explicit, r_explicit = item.split_messages(split=ConversationSplit.LAST_TURN) + assert [m.role for m in q_default] == [m.role for m in q_explicit] + assert [m.text for m in q_default] == [m.text for m in q_explicit] + assert [m.role for m in r_default] == [m.role for m in r_explicit] + assert [m.text for m in r_default] == [m.text for m in r_explicit] + + def test_per_turn_items_simple(self) -> None: + """per_turn_items produces one EvalItem per user message.""" + conversation = [ + Message("user", ["What's the weather?"]), + Message("assistant", ["It's 62°F."]), + Message("user", ["And tomorrow?"]), + Message("assistant", ["Rain expected."]), + ] + items = EvalItem.per_turn_items(conversation) + assert len(items) == 2 + + # Turn 1 + assert items[0].query == "What's the weather?" + assert items[0].response == "It's 62°F." + assert len(items[0].conversation) == 2 + + # Turn 2 — includes cumulative context; query joins all user texts in query split + assert items[1].query == "What's the weather? And tomorrow?" + assert items[1].response == "Rain expected." + assert len(items[1].conversation) == 4 + + def test_per_turn_items_with_tools(self) -> None: + """per_turn_items handles tool calls within a turn.""" + conversation = [ + Message("user", ["Check weather"]), + Message("assistant", [Content(type="function_call", name="get_weather")]), + Message("tool", [Content(type="function_result", result="sunny")]), + Message("assistant", ["It's sunny."]), + Message("user", ["Thanks"]), + Message("assistant", ["You're welcome!"]), + ] + tool_objs = [_make_tool("get_weather")] + items = EvalItem.per_turn_items(conversation, tools=tool_objs) + assert len(items) == 2 + + # Turn 1: response includes tool_call, tool_result, and final assistant + assert items[0].response == "It's sunny." + assert items[0].tools == tool_objs + assert len(items[0].conversation) == 4 # user, assistant(tool), tool, assistant + + # Turn 2 + assert items[1].response == "You're welcome!" + assert len(items[1].conversation) == 6 # full conversation + + def test_per_turn_items_empty(self) -> None: + """per_turn_items returns empty list when no user messages.""" + items = EvalItem.per_turn_items([Message("assistant", ["Hello"])]) + assert items == [] + + def test_per_turn_items_single_turn(self) -> None: + """per_turn_items with single turn produces one item.""" + conversation = [ + Message("user", ["Hi"]), + Message("assistant", ["Hello!"]), + ] + items = EvalItem.per_turn_items(conversation) + assert len(items) == 1 + assert items[0].query == "Hi" + assert items[0].response == "Hello!" + + def test_custom_splitter_callable(self) -> None: + """Custom callable splitter is used by split_messages().""" + conversation = [ + Message("user", ["Remember my name is Alice"]), + Message("assistant", ["Got it, Alice!"]), + Message("user", ["What's the capital of France?"]), + Message("assistant", [Content(type="function_call", name="retrieve_memory", call_id="m1")]), + Message("tool", [Content(type="function_result", call_id="m1", result="User name: Alice")]), + Message("assistant", ["The capital of France is Paris, Alice!"]), + ] + + def split_before_memory(conv): + """Split just before the memory retrieval tool call.""" + for i, msg in enumerate(conv): + for c in msg.contents: + if c.name == "retrieve_memory": + return conv[:i], conv[i:] + return EvalItem._split_last_turn_static(conv) + + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=split_before_memory) + + # split_before_memory finds "retrieve_memory" at conv[3] (assistant tool_call msg) + # query = conv[:3] = [user, assistant, user] + # response = conv[3:] = [assistant(tool_call), tool, assistant] + assert len(query_msgs) == 3 + assert query_msgs[-1].role == "user" + assert len(response_msgs) == 3 + assert response_msgs[0].role == "assistant" # the tool_call msg + + def test_custom_splitter_with_fallback(self) -> None: + """Custom splitter falls back to _split_last_turn_static when pattern not found.""" + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi there!"]), + ] + + def split_before_memory(conv): + for i, msg in enumerate(conv): + for c in msg.contents: + if c.name == "retrieve_memory": + return conv[:i], conv[i:] + return EvalItem._split_last_turn_static(conv) + + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=split_before_memory) + # Falls back to last-turn split + assert len(query_msgs) == 1 + assert query_msgs[0].role == "user" + assert len(response_msgs) == 1 + assert response_msgs[0].role == "assistant" + + def test_custom_splitter_lambda(self) -> None: + """A lambda works as a custom splitter.""" + conversation = [ + Message("user", ["A"]), + Message("assistant", ["B"]), + Message("user", ["C"]), + Message("assistant", ["D"]), + ] + # Split at index 2 (arbitrary) + item = EvalItem(conversation=conversation) + query_msgs, response_msgs = item.split_messages(split=lambda conv: (conv[:2], conv[2:])) + assert len(query_msgs) == 2 + assert len(response_msgs) == 2 + + def test_split_strategy_on_item_used_by_split_messages(self) -> None: + """split_strategy field on EvalItem is used as default by split_messages().""" + conversation = [ + Message("user", ["First"]), + Message("assistant", ["Response 1"]), + Message("user", ["Second"]), + Message("assistant", ["Response 2"]), + ] + item = EvalItem( + conversation=conversation, + split_strategy=ConversationSplit.FULL, + ) + # split_messages() with no split arg should use item.split_strategy + query_msgs, response_msgs = item.split_messages() + assert len(query_msgs) == 1 # FULL: just first user msg + assert query_msgs[0].text == "First" + assert len(response_msgs) == 3 + + def test_explicit_split_overrides_item_split_strategy(self) -> None: + """Explicit split= arg to split_messages() overrides item.split_strategy.""" + conversation = [ + Message("user", ["First"]), + Message("assistant", ["Response 1"]), + Message("user", ["Second"]), + Message("assistant", ["Response 2"]), + ] + item = EvalItem( + conversation=conversation, + split_strategy=ConversationSplit.FULL, + ) + # Explicit split= should override split_strategy + query_msgs, response_msgs = item.split_messages(split=ConversationSplit.LAST_TURN) + assert len(query_msgs) == 3 # LAST_TURN: up to last user + assert query_msgs[-1].text == "Second" + assert len(response_msgs) == 1 + + def test_no_split_defaults_to_last_turn(self) -> None: + """When neither split= nor split_strategy is set, defaults to LAST_TURN.""" + conversation = [ + Message("user", ["Hello"]), + Message("assistant", ["Hi"]), + ] + item = EvalItem(conversation=conversation) + assert item.split_strategy is None + query_msgs, response_msgs = item.split_messages() + assert len(query_msgs) == 1 + assert query_msgs[0].role == "user" + + +# --------------------------------------------------------------------------- +# _build_testing_criteria +# --------------------------------------------------------------------------- + + +class TestBuildTestingCriteria: + def test_without_data_mapping(self) -> None: + criteria = _build_testing_criteria(["relevance", "coherence"], "gpt-4o") + assert len(criteria) == 2 + assert criteria[0]["evaluator_name"] == "builtin.relevance" + assert criteria[0]["initialization_parameters"] == {"deployment_name": "gpt-4o"} + assert "data_mapping" not in criteria[0] + + def test_with_data_mapping(self) -> None: + criteria = _build_testing_criteria(["relevance", "groundedness"], "gpt-4o", include_data_mapping=True) + assert "data_mapping" in criteria[0] + # Quality evaluators should NOT have conversation + assert criteria[0]["data_mapping"] == { + "query": "{{item.query}}", + "response": "{{item.response}}", + } + # Groundedness has an extra context mapping + assert "context" in criteria[1]["data_mapping"] + assert "conversation" not in criteria[1]["data_mapping"] + + def test_tool_evaluator_includes_tool_definitions(self) -> None: + criteria = _build_testing_criteria(["relevance", "tool_call_accuracy"], "gpt-4o", include_data_mapping=True) + # relevance: string query/response + assert criteria[0]["data_mapping"]["query"] == "{{item.query}}" + assert criteria[0]["data_mapping"]["response"] == "{{item.response}}" + assert "tool_definitions" not in criteria[0]["data_mapping"] + # tool_call_accuracy: array query/response + tool_definitions + assert criteria[1]["data_mapping"]["query"] == "{{item.query_messages}}" + assert criteria[1]["data_mapping"]["response"] == "{{item.response_messages}}" + assert criteria[1]["data_mapping"]["tool_definitions"] == "{{item.tool_definitions}}" + + def test_agent_evaluators_use_message_arrays(self) -> None: + agent_evals = ["task_adherence", "intent_resolution", "task_completion"] + criteria = _build_testing_criteria(agent_evals, "gpt-4o", include_data_mapping=True) + for c in criteria: + assert c["data_mapping"]["query"] == "{{item.query_messages}}", f"{c['name']}" + assert c["data_mapping"]["response"] == "{{item.response_messages}}", f"{c['name']}" + + def test_quality_evaluators_use_strings(self) -> None: + quality_evals = ["coherence", "relevance", "fluency"] + criteria = _build_testing_criteria(quality_evals, "gpt-4o", include_data_mapping=True) + for c in criteria: + assert c["data_mapping"]["query"] == "{{item.query}}", f"{c['name']}" + assert c["data_mapping"]["response"] == "{{item.response}}", f"{c['name']}" + + def test_all_tool_evaluators_include_tool_definitions(self) -> None: + tool_evals = [ + "tool_call_accuracy", + "tool_selection", + "tool_input_accuracy", + "tool_output_utilization", + "tool_call_success", + ] + criteria = _build_testing_criteria(tool_evals, "gpt-4o", include_data_mapping=True) + for c in criteria: + assert "tool_definitions" in c["data_mapping"], f"{c['name']} missing tool_definitions" + + +# --------------------------------------------------------------------------- +# _build_item_schema +# --------------------------------------------------------------------------- + + +class TestBuildItemSchema: + def test_without_context(self) -> None: + schema = _build_item_schema(has_context=False) + assert "context" not in schema["properties"] + assert schema["required"] == ["query", "response"] + + def test_with_context(self) -> None: + schema = _build_item_schema(has_context=True) + assert "context" in schema["properties"] + + def test_with_tools(self) -> None: + schema = _build_item_schema(has_tools=True) + assert "tool_definitions" in schema["properties"] + + def test_with_context_and_tools(self) -> None: + schema = _build_item_schema(has_context=True, has_tools=True) + assert "context" in schema["properties"] + assert "tool_definitions" in schema["properties"] + + +# --------------------------------------------------------------------------- +# FoundryEvals (constructor, name, select, evaluate via dataset) +# --------------------------------------------------------------------------- + + +class TestFoundryEvals: + def test_constructor_with_openai_client(self) -> None: + mock_client = MagicMock() + fe = FoundryEvals(client=mock_client, model="gpt-4o") + assert fe.name == "Microsoft Foundry" + + def test_constructor_with_project_client(self) -> None: + mock_oai = MagicMock(spec=AsyncOpenAI) + mock_project = MagicMock() + mock_project.get_openai_client.return_value = mock_oai + fe = FoundryEvals(project_client=mock_project, model="gpt-4o") + assert fe.name == "Microsoft Foundry" + mock_project.get_openai_client.assert_called_once() + + def test_constructor_no_client_auto_creates_from_env(self) -> None: + """When no client/project_client given, auto-creates FoundryChatClient from env.""" + import os + from unittest.mock import patch + + with patch.dict(os.environ, {}, clear=True), pytest.raises((ValueError, Exception)): + FoundryEvals(model="gpt-4o") + + def test_name_property(self) -> None: + fe = FoundryEvals(client=MagicMock(), model="gpt-4o") + assert fe.name == "Microsoft Foundry" + + def test_evaluators_passed_in_constructor(self) -> None: + fe = FoundryEvals( + client=MagicMock(), + model="gpt-4o", + evaluators=["relevance", "coherence"], + ) + assert fe._evaluators == ["relevance", "coherence"] + + async def test_evaluate_calls_evals_api(self) -> None: + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_123" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_456" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=2) + mock_completed.report_url = "https://portal.azure.com/eval/run_456" + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + # Mock output_items.list so _fetch_output_items exercises the full flow + mock_output_item = MagicMock() + mock_output_item.id = "output_item_1" + mock_output_item.status = "pass" + mock_output_item.sample = MagicMock(error=None, usage=None, input=[], output=[]) + mock_result = MagicMock(status="pass", score=5, reason="Relevant response") + mock_result.name = "relevance" # MagicMock(name=...) sets display name, not .name attr + mock_output_item.results = [mock_result] + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_output_item])) + + items = [ + EvalItem(conversation=[Message("user", ["Hello"]), Message("assistant", ["Hi there!"])]), + EvalItem(conversation=[Message("user", ["Weather?"]), Message("assistant", ["Sunny."])]), + ] + + fe = FoundryEvals( + client=mock_client, + model="gpt-4o", + evaluators=[FoundryEvals.RELEVANCE], + ) + results = await fe.evaluate(items) + + assert isinstance(results, EvalResults) + assert results.status == "completed" + assert results.eval_id == "eval_123" + assert results.run_id == "run_456" + assert results.report_url == "https://portal.azure.com/eval/run_456" + assert results.all_passed + assert results.passed == 2 + assert results.failed == 0 + + # Verify per-item output_items were fetched + assert len(results.items) == 1 + assert results.items[0].item_id == "output_item_1" + assert results.items[0].status == "pass" + assert len(results.items[0].scores) == 1 + assert results.items[0].scores[0].name == "relevance" + assert results.items[0].scores[0].score == 5 + + # Verify evals.create was called with correct structure + create_call = mock_client.evals.create.call_args + assert create_call.kwargs["name"] == "Agent Framework Eval" + assert create_call.kwargs["data_source_config"]["type"] == "custom" + + # Verify evals.runs.create was called with JSONL data source + run_call = mock_client.evals.runs.create.call_args + assert run_call.kwargs["data_source"]["type"] == "jsonl" + content = run_call.kwargs["data_source"]["source"]["content"] + assert len(content) == 2 + + async def test_evaluate_uses_default_evaluators(self) -> None: + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_1" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_1" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + fe = FoundryEvals(client=mock_client, model="gpt-4o") + await fe.evaluate([EvalItem(conversation=[Message("user", ["Hi"]), Message("assistant", ["Hello"])])]) + + # Verify default evaluators were used + create_call = mock_client.evals.create.call_args + criteria = create_call.kwargs["testing_criteria"] + names = {c["name"] for c in criteria} + assert "relevance" in names + assert "coherence" in names + assert "task_adherence" in names + + async def test_evaluate_uses_dataset_path(self) -> None: + """Items use the JSONL dataset path.""" + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_ds" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_ds" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + items = [ + EvalItem( + conversation=[Message("user", ["What's the weather?"]), Message("assistant", ["Sunny"])], + ), + ] + + fe = FoundryEvals(client=mock_client, model="gpt-4o") + await fe.evaluate(items) + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + content = ds["source"]["content"] + assert content[0]["item"]["query"] == "What's the weather?" + + async def test_evaluate_with_tool_items_uses_dataset_path(self) -> None: + """Items with tool_definitions use the dataset path.""" + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tool" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tool" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + items = [ + EvalItem( + conversation=[Message("user", ["Do the thing"]), Message("assistant", ["Done"])], + tools=[_make_tool("my_tool")], + ), + ] + + fe = FoundryEvals( + client=mock_client, + model="gpt-4o", + evaluators=[FoundryEvals.TOOL_CALL_ACCURACY], + ) + await fe.evaluate(items) + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + assert "tool_definitions" in ds["source"]["content"][0]["item"] + + async def test_evaluate_image_content_in_dataset(self) -> None: + """Image content in conversations is preserved in the JSONL payload.""" + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_img" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_img" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + items = [ + EvalItem( + conversation=[ + Message( + "user", + [ + Content.from_text("Describe this image"), + Content.from_uri("https://example.com/photo.jpg", media_type="image/jpeg"), + ], + ), + Message("assistant", ["A beautiful sunset over the ocean."]), + ], + ), + ] + + fe = FoundryEvals(client=mock_client, model="gpt-4o") + await fe.evaluate(items) + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + item_data = ds["source"]["content"][0]["item"] + + # query_messages should contain the image + query_msgs = item_data["query_messages"] + user_msg = query_msgs[0] + assert user_msg["role"] == "user" + assert len(user_msg["content"]) == 2 + assert user_msg["content"][0] == {"type": "text", "text": "Describe this image"} + assert user_msg["content"][1]["type"] == "input_image" + assert user_msg["content"][1]["image_url"] == "https://example.com/photo.jpg" + + async def test_evaluate_with_project_client(self) -> None: + mock_oai = MagicMock(spec=AsyncOpenAI) + mock_project = MagicMock() + mock_project.get_openai_client.return_value = mock_oai + + mock_eval = MagicMock() + mock_eval.id = "eval_pc" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_pc" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + fe = FoundryEvals(project_client=mock_project, model="gpt-4o") + results = await fe.evaluate([EvalItem(conversation=[Message("user", ["Hi"]), Message("assistant", ["Hello"])])]) + + assert results.status == "completed" + mock_project.get_openai_client.assert_called_once() + + +# --------------------------------------------------------------------------- +# FoundryEvals constants +# --------------------------------------------------------------------------- + + +class TestEvaluators: + def test_constants_resolve(self) -> None: + assert _resolve_evaluator(FoundryEvals.RELEVANCE) == "builtin.relevance" + assert _resolve_evaluator(FoundryEvals.TOOL_CALL_ACCURACY) == "builtin.tool_call_accuracy" + assert _resolve_evaluator(FoundryEvals.VIOLENCE) == "builtin.violence" + assert _resolve_evaluator(FoundryEvals.INTENT_RESOLUTION) == "builtin.intent_resolution" + + def test_all_constants_are_valid(self) -> None: + for attr in dir(FoundryEvals): + if attr.startswith("_"): + continue + value = getattr(FoundryEvals, attr) + if isinstance(value, str): + _resolve_evaluator(value) # should not raise + + +# --------------------------------------------------------------------------- +# _resolve_default_evaluators +# --------------------------------------------------------------------------- + + +class TestResolveDefaultEvaluators: + def test_explicit_evaluators_passthrough(self) -> None: + result = _resolve_default_evaluators([FoundryEvals.VIOLENCE]) + assert result == [FoundryEvals.VIOLENCE] + + def test_none_gives_defaults(self) -> None: + result = _resolve_default_evaluators(None) + assert FoundryEvals.RELEVANCE in result + assert FoundryEvals.COHERENCE in result + assert FoundryEvals.TASK_ADHERENCE in result + assert FoundryEvals.TOOL_CALL_ACCURACY not in result + + def test_none_with_tool_items_adds_tool_eval(self) -> None: + items = [ + EvalItem( + conversation=[Message("user", ["search for stuff"]), Message("assistant", ["found it"])], + tools=[_make_tool("search")], + ), + ] + result = _resolve_default_evaluators(None, items=items) + assert FoundryEvals.TOOL_CALL_ACCURACY in result + + def test_explicit_evaluators_ignore_tool_items(self) -> None: + items = [ + EvalItem( + conversation=[Message("user", ["search"]), Message("assistant", ["found"])], + tools=[_make_tool("search")], + ), + ] + result = _resolve_default_evaluators([FoundryEvals.RELEVANCE], items=items) + assert result == [FoundryEvals.RELEVANCE] + + +# --------------------------------------------------------------------------- +# _filter_tool_evaluators +# --------------------------------------------------------------------------- + + +class TestFilterToolEvaluators: + def test_keeps_tool_evaluators_when_items_have_tools(self) -> None: + items = [ + EvalItem(conversation=[Message("user", ["q"]), Message("assistant", ["r"])], tools=[_make_tool("t")]), + ] + result = _filter_tool_evaluators( + ["relevance", "tool_call_accuracy"], + items, + ) + assert "relevance" in result + assert "tool_call_accuracy" in result + + def test_removes_tool_evaluators_when_no_tools(self) -> None: + items = [ + EvalItem(conversation=[Message("user", ["q"]), Message("assistant", ["r"])]), + ] + result = _filter_tool_evaluators( + ["relevance", "tool_call_accuracy"], + items, + ) + assert "relevance" in result + assert "tool_call_accuracy" not in result + + def test_raises_when_all_filtered(self) -> None: + items = [ + EvalItem(conversation=[Message("user", ["q"]), Message("assistant", ["r"])]), + ] + with pytest.raises(ValueError, match="require tool definitions"): + _filter_tool_evaluators( + ["tool_call_accuracy", "tool_selection"], + items, + ) + + +# --------------------------------------------------------------------------- +# EvalResults +# --------------------------------------------------------------------------- + + +class TestEvalResults: + def test_all_passed_true(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 3, "failed": 0, "errored": 0}, + ) + assert r.all_passed + assert r.passed == 3 + assert r.failed == 0 + assert r.total == 3 + + def test_all_passed_false_on_failure(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 2, "failed": 1, "errored": 0}, + ) + assert not r.all_passed + assert r.failed == 1 + + def test_all_passed_false_on_error(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 2, "failed": 0, "errored": 1}, + ) + assert not r.all_passed + + def test_all_passed_false_on_non_completed(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="timeout", + result_counts={"passed": 2, "failed": 0, "errored": 0}, + ) + assert not r.all_passed + + def test_all_passed_false_on_empty(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 0, "failed": 0, "errored": 0}, + ) + assert not r.all_passed + + def test_raise_for_status_succeeds(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 1, "failed": 0, "errored": 0}, + ) + r.raise_for_status() # should not raise + + def test_raise_for_status_raises(self) -> None: + r = EvalResults( + provider="test", + eval_id="e", + run_id="r", + status="completed", + result_counts={"passed": 1, "failed": 1, "errored": 0}, + ) + with pytest.raises(EvalNotPassedError, match="1 passed, 1 failed"): + r.raise_for_status() + + def test_raise_for_status_custom_message(self) -> None: + r = EvalResults(provider="test", eval_id="e", run_id="r", status="failed") + with pytest.raises(EvalNotPassedError, match="custom error"): + r.raise_for_status("custom error") + + def test_none_result_counts(self) -> None: + r = EvalResults(provider="test", eval_id="e", run_id="r", status="completed") + assert r.passed == 0 + assert r.failed == 0 + assert r.total == 0 + assert not r.all_passed + + +# --------------------------------------------------------------------------- +# _resolve_openai_client +# --------------------------------------------------------------------------- + + +class TestResolveOpenAIClient: + def test_explicit_client(self) -> None: + mock_client = MagicMock() + assert _resolve_openai_client(client=mock_client) is mock_client + + def test_project_client(self) -> None: + mock_oai = MagicMock(spec=AsyncOpenAI) + mock_project = MagicMock() + mock_project.get_openai_client.return_value = mock_oai + + result = _resolve_openai_client(project_client=mock_project) + assert result is mock_oai + mock_project.get_openai_client.assert_called_once() + + def test_explicit_takes_precedence(self) -> None: + mock_client = MagicMock() + mock_project = MagicMock() + + result = _resolve_openai_client(client=mock_client, project_client=mock_project) + assert result is mock_client + mock_project.get_openai_client.assert_not_called() + + def test_neither_raises(self) -> None: + with pytest.raises(ValueError, match="Provide either"): + _resolve_openai_client() + + +# --------------------------------------------------------------------------- +# evaluate_agent with responses= (core function, uses FoundryEvals as evaluator) +# --------------------------------------------------------------------------- + + +class TestEvaluateAgentWithResponses: + async def test_responses_without_queries_raises(self) -> None: + mock_oai = MagicMock() + response = AgentResponse(messages=[Message("assistant", ["Hello"])]) + + with pytest.raises(ValueError, match="Provide 'queries' alongside 'responses'"): + await evaluate_agent( + responses=response, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + async def test_fallback_to_dataset_with_query(self) -> None: + """Non-Responses-API: falls back to dataset path when query is provided.""" + mock_oai = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_fb" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_fb" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = "https://portal.azure.com/eval" + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + response = AgentResponse(messages=[Message("assistant", ["It's sunny."])]) + + results = await evaluate_agent( + responses=response, + queries=["What's the weather?"], + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + assert results[0].status == "completed" + assert results[0].all_passed + + # Should use jsonl data source (dataset path), not azure_ai_responses + run_call = mock_oai.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + content = ds["source"]["content"] + assert len(content) == 1 + assert content[0]["item"]["query"] == "What's the weather?" + assert content[0]["item"]["response"] == "It's sunny." + + async def test_fallback_with_agent_extracts_tools(self) -> None: + """Non-Responses-API with agent: tool definitions are included in the eval item.""" + mock_oai = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tools" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tools" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + mock_agent = MagicMock() + mock_agent.default_options = { + "tools": [FunctionTool(name="my_tool", description="A test tool", func=lambda x: x)] + } + + response = AgentResponse(messages=[Message("assistant", ["Result."])]) + + results = await evaluate_agent( + responses=response, + queries=["Do the thing"], + agent=mock_agent, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + assert results[0].status == "completed" + + run_call = mock_oai.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + content = ds["source"]["content"] + item = content[0]["item"] + assert "tool_definitions" in item + tool_defs = item["tool_definitions"] + assert any(t["name"] == "my_tool" for t in tool_defs) + + async def test_fallback_multiple_responses_with_queries(self) -> None: + """Non-Responses-API with multiple responses requires matching queries.""" + mock_oai = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_multi_fb" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_multi_fb" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=2) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + responses = [ + AgentResponse(messages=[Message("assistant", ["Answer 1"])]), + AgentResponse(messages=[Message("assistant", ["Answer 2"])]), + ] + + results = await evaluate_agent( + responses=responses, + queries=["Question 1", "Question 2"], + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + assert results[0].passed == 2 + run_call = mock_oai.evals.runs.create.call_args + content = run_call.kwargs["data_source"]["source"]["content"] + assert len(content) == 2 + assert content[0]["item"]["query"] == "Question 1" + assert content[1]["item"]["query"] == "Question 2" + + async def test_query_response_count_mismatch_raises(self) -> None: + """Mismatched query and response counts should raise.""" + mock_oai = MagicMock() + + responses = [ + AgentResponse(messages=[Message("assistant", ["A1"])]), + AgentResponse(messages=[Message("assistant", ["A2"])]), + ] + + with pytest.raises(ValueError, match="queries but"): + await evaluate_agent( + responses=responses, + queries=["Q1", "Q2", "Q3"], + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + async def test_tool_evaluators_with_query_and_agent_uses_dataset_path(self) -> None: + """Tool evaluators with query+agent uses dataset path.""" + mock_oai = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tool" + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tool" + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + response = AgentResponse( + messages=[Message("assistant", ["It's sunny"])], + ) + + agent = MagicMock() + agent.default_options = { + "tools": [ + FunctionTool(name="get_weather", description="Get weather", func=lambda: None), + ] + } + + fe = FoundryEvals( + client=mock_oai, + model="gpt-4o", + evaluators=[FoundryEvals.TOOL_CALL_ACCURACY], + ) + + await evaluate_agent( + responses=response, + queries=["What's the weather?"], + agent=agent, + evaluators=fe, + ) + + # Verify it used the dataset path (jsonl), not Responses API path + run_call = mock_oai.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "jsonl" + + # Verify tool_definitions are in the data items + items = ds["source"]["content"] + assert "tool_definitions" in items[0]["item"] + + +# --------------------------------------------------------------------------- +# EvalResults.sub_results +# --------------------------------------------------------------------------- + + +class TestEvalResultsSubResults: + def test_sub_results_default_empty(self) -> None: + r = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 1, "failed": 0}, + ) + assert r.sub_results == {} + assert r.all_passed + + def test_all_passed_checks_sub_results(self) -> None: + parent = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0}, + sub_results={ + "agent-a": EvalResults( + provider="test", + eval_id="e2", + run_id="r2", + status="completed", + result_counts={"passed": 1, "failed": 0}, + ), + "agent-b": EvalResults( + provider="test", + eval_id="e3", + run_id="r3", + status="completed", + result_counts={"passed": 1, "failed": 1}, + ), + }, + ) + assert not parent.all_passed # agent-b has a failure + + def test_all_passed_with_all_sub_passing(self) -> None: + parent = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0}, + sub_results={ + "agent-a": EvalResults( + provider="test", + eval_id="e2", + run_id="r2", + status="completed", + result_counts={"passed": 1, "failed": 0}, + ), + }, + ) + assert parent.all_passed + + def test_raise_for_status_includes_failed_agents(self) -> None: + parent = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 0}, + sub_results={ + "good-agent": EvalResults( + provider="test", + eval_id="e2", + run_id="r2", + status="completed", + result_counts={"passed": 1, "failed": 0}, + ), + "bad-agent": EvalResults( + provider="test", + eval_id="e3", + run_id="r3", + status="completed", + result_counts={"passed": 0, "failed": 1}, + ), + }, + ) + with pytest.raises(EvalNotPassedError, match="bad-agent"): + parent.raise_for_status() + + +# --------------------------------------------------------------------------- +# _extract_agent_eval_data +# --------------------------------------------------------------------------- + + +def _make_agent_exec_response( + executor_id: str, + response_text: str, + user_messages: list[str] | None = None, +) -> AgentExecutorResponse: + """Helper to build an AgentExecutorResponse for testing.""" + agent_response = AgentResponse(messages=[Message("assistant", [response_text])]) + full_conv: list[Message] = [] + if user_messages: + for m in user_messages: + full_conv.append(Message("user", [m])) + full_conv.extend(agent_response.messages) + return AgentExecutorResponse( + executor_id=executor_id, + agent_response=agent_response, + full_conversation=full_conv, + ) + + +class TestExtractAgentEvalData: + def test_extracts_single_agent(self) -> None: + aer = _make_agent_exec_response("planner", "Plan is ready", ["Plan a trip"]) + + events = [ + WorkflowEvent.executor_invoked("planner", "Plan a trip"), + WorkflowEvent.executor_completed("planner", [aer]), + ] + result = WorkflowRunResult(events, []) + + data = _extract_agent_eval_data(result) + assert len(data) == 1 + assert data[0]["executor_id"] == "planner" + assert data[0]["response"].text == "Plan is ready" + + def test_extracts_multiple_agents(self) -> None: + aer1 = _make_agent_exec_response("planner", "Plan done", ["Plan a trip"]) + aer2 = _make_agent_exec_response("booker", "Booked!", ["Book flight"]) + + events = [ + WorkflowEvent.executor_invoked("planner", "Plan a trip"), + WorkflowEvent.executor_completed("planner", [aer1]), + WorkflowEvent.executor_invoked("booker", "Book flight"), + WorkflowEvent.executor_completed("booker", [aer2]), + ] + result = WorkflowRunResult(events, []) + + data = _extract_agent_eval_data(result) + assert len(data) == 2 + assert data[0]["executor_id"] == "planner" + assert data[1]["executor_id"] == "booker" + + def test_skips_internal_executors(self) -> None: + aer = _make_agent_exec_response("planner", "Done", ["Go"]) + + events = [ + WorkflowEvent.executor_invoked("input-conversation", "hello"), + WorkflowEvent.executor_completed("input-conversation", ["hello"]), + WorkflowEvent.executor_invoked("planner", "Go"), + WorkflowEvent.executor_completed("planner", [aer]), + WorkflowEvent.executor_invoked("end", []), + WorkflowEvent.executor_completed("end", None), + ] + result = WorkflowRunResult(events, []) + + data = _extract_agent_eval_data(result) + assert len(data) == 1 + assert data[0]["executor_id"] == "planner" + + def test_resolves_agent_from_workflow(self) -> None: + aer = _make_agent_exec_response("my-agent", "Done", ["Do it"]) + + events = [ + WorkflowEvent.executor_invoked("my-agent", "Do it"), + WorkflowEvent.executor_completed("my-agent", [aer]), + ] + result = WorkflowRunResult(events, []) + + # Build a mock workflow with AgentExecutor + from agent_framework import AgentExecutor + + mock_agent = MagicMock() + mock_agent.default_options = {"tools": []} + mock_executor = MagicMock(spec=AgentExecutor) + mock_executor.agent = mock_agent + + mock_workflow = MagicMock() + mock_workflow.executors = {"my-agent": mock_executor} + + data = _extract_agent_eval_data(result, mock_workflow) + assert len(data) == 1 + assert data[0]["agent"] is mock_agent + + +class TestExtractOverallQuery: + def test_extracts_string_query(self) -> None: + events = [WorkflowEvent.executor_invoked("input", "Plan a trip")] + result = WorkflowRunResult(events, []) + assert _extract_overall_query(result) == "Plan a trip" + + def test_extracts_message_query(self) -> None: + msgs = [Message("user", ["What's the weather?"])] + events = [WorkflowEvent.executor_invoked("input", msgs)] + result = WorkflowRunResult(events, []) + assert "What's the weather?" in (_extract_overall_query(result) or "") + + def test_returns_none_for_empty(self) -> None: + result = WorkflowRunResult([], []) + assert _extract_overall_query(result) is None + + +# --------------------------------------------------------------------------- +# evaluate_workflow (core function, uses FoundryEvals as evaluator) +# --------------------------------------------------------------------------- + + +class TestEvaluateWorkflow: + def _mock_oai_client(self, eval_id: str = "eval_wf", run_id: str = "run_wf") -> MagicMock: + mock_oai = MagicMock() + mock_eval = MagicMock() + mock_eval.id = eval_id + mock_oai.evals.create = AsyncMock(return_value=mock_eval) + mock_run = MagicMock() + mock_run.id = run_id + mock_oai.evals.runs.create = AsyncMock(return_value=mock_run) + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = "https://portal.azure.com/eval" + mock_completed.per_testing_criteria_results = None + mock_oai.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + return mock_oai + + async def test_post_hoc_with_workflow_result(self) -> None: + """Evaluate a workflow result that was already produced.""" + mock_oai = self._mock_oai_client() + + aer1 = _make_agent_exec_response("writer", "Draft written", ["Write about Paris"]) + aer2 = _make_agent_exec_response("reviewer", "Looks good!", ["Review: Draft written"]) + + final_output = [Message("assistant", ["Final reviewed output"])] + + events = [ + WorkflowEvent.executor_invoked("input-conversation", "Write about Paris"), + WorkflowEvent.executor_completed("input-conversation", None), + WorkflowEvent.executor_invoked("writer", "Write about Paris"), + WorkflowEvent.executor_completed("writer", [aer1]), + WorkflowEvent.executor_invoked("reviewer", [aer1]), + WorkflowEvent.executor_completed("reviewer", [aer2]), + WorkflowEvent.output("end", final_output), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + + results = await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + include_overall=False, + ) + + assert results[0].status == "completed" + assert "writer" in results[0].sub_results + assert "reviewer" in results[0].sub_results + assert len(results[0].sub_results) == 2 + + async def test_with_queries_runs_workflow(self) -> None: + """Passing queries= runs the workflow and evaluates.""" + mock_oai = self._mock_oai_client() + + aer = _make_agent_exec_response("agent", "Response", ["Query"]) + final_output = [Message("assistant", ["Final"])] + + events = [ + WorkflowEvent.executor_invoked("agent", "Test query"), + WorkflowEvent.executor_completed("agent", [aer]), + WorkflowEvent.output("end", final_output), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + mock_workflow.run = AsyncMock(return_value=wf_result) + + results = await evaluate_workflow( + workflow=mock_workflow, + queries=["Test query"], + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + include_overall=False, + ) + + mock_workflow.run.assert_called_once_with("Test query") + assert "agent" in results[0].sub_results + + async def test_overall_plus_per_agent(self) -> None: + """Both overall and per-agent evals run by default.""" + mock_oai = self._mock_oai_client() + + aer = _make_agent_exec_response("planner", "Plan done", ["Plan trip"]) + final_output = [Message("assistant", ["Trip planned!"])] + + events = [ + WorkflowEvent.executor_invoked("input-conversation", "Plan trip"), + WorkflowEvent.executor_completed("input-conversation", None), + WorkflowEvent.executor_invoked("planner", "Plan trip"), + WorkflowEvent.executor_completed("planner", [aer]), + WorkflowEvent.output("end", final_output), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + + results = await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + # Should have per-agent sub_results AND overall + assert "planner" in results[0].sub_results + assert results[0].status == "completed" + # FoundryEvals.evaluate called twice: once for planner, once for overall + assert mock_oai.evals.create.call_count == 2 + + async def test_no_result_or_queries_raises(self) -> None: + mock_oai = MagicMock() + mock_workflow = MagicMock() + + with pytest.raises(ValueError, match="Provide either"): + await evaluate_workflow( + workflow=mock_workflow, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + ) + + async def test_per_agent_only(self) -> None: + """include_overall=False skips the overall eval.""" + mock_oai = self._mock_oai_client() + + aer = _make_agent_exec_response("agent-a", "Done", ["Do stuff"]) + + events = [ + WorkflowEvent.executor_invoked("agent-a", "Do stuff"), + WorkflowEvent.executor_completed("agent-a", [aer]), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + + results = await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=FoundryEvals(client=mock_oai, model="gpt-4o"), + include_overall=False, + ) + + assert "agent-a" in results[0].sub_results + # Only one eval call (per-agent), no overall + assert mock_oai.evals.create.call_count == 1 + + async def test_overall_eval_excludes_tool_evaluators(self) -> None: + """Tool evaluators should not be passed to the overall workflow eval.""" + mock_oai = self._mock_oai_client() + + aer = _make_agent_exec_response("researcher", "Weather is sunny", ["What's the weather?"]) + + events = [ + WorkflowEvent.executor_invoked("input-conversation", "What's the weather?"), + WorkflowEvent.executor_completed("input-conversation", None), + WorkflowEvent.executor_invoked("researcher", "What's the weather?"), + WorkflowEvent.executor_completed("researcher", [aer]), + WorkflowEvent.output("end", [Message("assistant", ["Weather is sunny"])]), + ] + wf_result = WorkflowRunResult(events, []) + + mock_workflow = MagicMock() + mock_workflow.executors = {} + + fe = FoundryEvals( + client=mock_oai, + model="gpt-4o", + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY], + ) + + await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=fe, + ) + + # Should have 2 evals: one per-agent, one overall + assert mock_oai.evals.create.call_count == 2 + + # Check the overall eval's testing_criteria doesn't include tool_call_accuracy + overall_call = mock_oai.evals.create.call_args_list[-1] + overall_criteria = overall_call.kwargs["testing_criteria"] + evaluator_names = [c["evaluator_name"] for c in overall_criteria] + assert "builtin.tool_call_accuracy" not in evaluator_names + assert "builtin.relevance" in evaluator_names + + async def test_per_agent_excludes_tool_evaluators_when_no_tools(self) -> None: + """Sub-agents without tools should not get tool evaluators.""" + mock_oai = self._mock_oai_client() + + # researcher has tools, planner does not + aer1 = _make_agent_exec_response("researcher", "Weather is sunny", ["Check weather"]) + aer2 = _make_agent_exec_response("planner", "Trip planned", ["Plan based on: sunny"]) + + events = [ + WorkflowEvent.executor_invoked("researcher", "Check weather"), + WorkflowEvent.executor_completed("researcher", [aer1]), + WorkflowEvent.executor_invoked("planner", "Plan based on: sunny"), + WorkflowEvent.executor_completed("planner", [aer2]), + ] + wf_result = WorkflowRunResult(events, []) + + from agent_framework import AgentExecutor + + # researcher has tools + mock_researcher = MagicMock() + mock_researcher.default_options = { + "tools": [ + FunctionTool(name="get_weather", description="Get weather", func=lambda: None), + ] + } + mock_researcher_executor = MagicMock(spec=AgentExecutor) + mock_researcher_executor.agent = mock_researcher + + # planner has NO tools + mock_planner = MagicMock() + mock_planner.default_options = {"tools": []} + mock_planner_executor = MagicMock(spec=AgentExecutor) + mock_planner_executor.agent = mock_planner + + mock_workflow = MagicMock() + mock_workflow.executors = { + "researcher": mock_researcher_executor, + "planner": mock_planner_executor, + } + + fe = FoundryEvals( + client=mock_oai, + model="gpt-4o", + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY], + ) + + await evaluate_workflow( + workflow=mock_workflow, + workflow_result=wf_result, + evaluators=fe, + include_overall=False, + ) + + # Two sub-agent evals + assert mock_oai.evals.create.call_count == 2 + + # Find which call is for researcher vs planner by eval name + for call in mock_oai.evals.create.call_args_list: + criteria = call.kwargs["testing_criteria"] + eval_names = [c["evaluator_name"] for c in criteria] + name = call.kwargs["name"] + if "planner" in name: + assert "builtin.tool_call_accuracy" not in eval_names, ( + "planner has no tools — should not get tool_call_accuracy" + ) + elif "researcher" in name: + assert "builtin.tool_call_accuracy" in eval_names, ( + "researcher has tools — should get tool_call_accuracy" + ) + + +# --------------------------------------------------------------------------- +# EvalItemResult and EvalScoreResult +# --------------------------------------------------------------------------- + + +class TestEvalItemResult: + def test_status_properties(self) -> None: + from agent_framework._evaluation import EvalItemResult + + passed = EvalItemResult(item_id="1", status="pass") + assert passed.is_passed + assert not passed.is_failed + assert not passed.is_error + + failed = EvalItemResult(item_id="2", status="fail") + assert not failed.is_passed + assert failed.is_failed + assert not failed.is_error + + errored = EvalItemResult(item_id="3", status="error") + assert not errored.is_passed + assert not errored.is_failed + assert errored.is_error + + errored2 = EvalItemResult(item_id="4", status="errored") + assert errored2.is_error + + def test_with_scores(self) -> None: + from agent_framework._evaluation import EvalItemResult, EvalScoreResult + + scores = [ + EvalScoreResult(name="relevance", score=0.9, passed=True), + EvalScoreResult(name="coherence", score=0.3, passed=False), + ] + item = EvalItemResult(item_id="1", status="fail", scores=scores) + assert len(item.scores) == 2 + assert item.scores[0].passed is True + assert item.scores[1].passed is False + + def test_with_error(self) -> None: + from agent_framework._evaluation import EvalItemResult + + item = EvalItemResult( + item_id="1", + status="error", + error_code="QueryExtractionError", + error_message="Query list cannot be empty", + ) + assert item.is_error + assert item.error_code == "QueryExtractionError" + + def test_with_token_usage(self) -> None: + from agent_framework._evaluation import EvalItemResult + + item = EvalItemResult( + item_id="1", + status="pass", + token_usage={"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + ) + assert item.token_usage is not None + assert item.token_usage["total_tokens"] == 150 + + +class TestEvalResultsWithItems: + def test_item_status_properties(self) -> None: + from agent_framework._evaluation import EvalItemResult + + results = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 2, "failed": 1, "errored": 1}, + items=[ + EvalItemResult(item_id="1", status="pass"), + EvalItemResult(item_id="2", status="pass"), + EvalItemResult(item_id="3", status="fail"), + EvalItemResult(item_id="4", status="error", error_code="QueryExtractionError"), + ], + ) + assert sum(1 for i in results.items if i.is_passed) == 2 + assert sum(1 for i in results.items if i.is_failed) == 1 + assert sum(1 for i in results.items if i.is_error) == 1 + + def test_raise_for_status_includes_errored_items(self) -> None: + from agent_framework._evaluation import EvalItemResult + + results = EvalResults( + provider="test", + eval_id="e1", + run_id="r1", + status="completed", + result_counts={"passed": 0, "failed": 0, "errored": 2}, + items=[ + EvalItemResult(item_id="i1", status="error", error_code="QueryExtractionError"), + EvalItemResult(item_id="i2", status="error", error_code="TimeoutError"), + ], + ) + with pytest.raises(EvalNotPassedError, match="Errored items: i1: QueryExtractionError"): + results.raise_for_status() + + +# --------------------------------------------------------------------------- +# _fetch_output_items +# --------------------------------------------------------------------------- + + +class TestFetchOutputItems: + async def test_fetches_and_converts_output_items(self) -> None: + from agent_framework_foundry._foundry_evals import _fetch_output_items + + # Build mock output items matching the OpenAI SDK schema + mock_result = MagicMock() + mock_result.name = "relevance" + mock_result.score = 0.85 + mock_result.passed = True + mock_result.sample = None + + mock_usage = MagicMock() + mock_usage.prompt_tokens = 100 + mock_usage.completion_tokens = 50 + mock_usage.total_tokens = 150 + mock_usage.cached_tokens = 0 + + mock_input = MagicMock() + mock_input.role = "user" + mock_input.content = "What is the weather?" + + mock_output = MagicMock() + mock_output.role = "assistant" + mock_output.content = "It is sunny." + + mock_error = MagicMock() + mock_error.code = "" + mock_error.message = "" + + mock_sample = MagicMock() + mock_sample.error = mock_error + mock_sample.usage = mock_usage + mock_sample.input = [mock_input] + mock_sample.output = [mock_output] + + mock_oi = MagicMock() + mock_oi.id = "oi_abc123" + mock_oi.status = "pass" + mock_oi.results = [mock_result] + mock_oi.sample = mock_sample + mock_oi.datasource_item = {"resp_id": "resp_xyz"} + + mock_client = MagicMock() + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_oi])) + + items = await _fetch_output_items(mock_client, "eval_1", "run_1") + + assert len(items) == 1 + item = items[0] + assert item.item_id == "oi_abc123" + assert item.status == "pass" + assert item.is_passed + assert len(item.scores) == 1 + assert item.scores[0].name == "relevance" + assert item.scores[0].score == 0.85 + assert item.scores[0].passed is True + assert item.response_id == "resp_xyz" + assert item.input_text == "What is the weather?" + assert item.output_text == "It is sunny." + assert item.token_usage is not None + assert item.token_usage["total_tokens"] == 150 + assert item.error_code is None + + async def test_handles_errored_item(self) -> None: + from agent_framework_foundry._foundry_evals import _fetch_output_items + + mock_error = MagicMock() + mock_error.code = "QueryExtractionError" + mock_error.message = "Query list cannot be empty" + + mock_sample = MagicMock() + mock_sample.error = mock_error + mock_sample.usage = None + mock_sample.input = [] + mock_sample.output = [] + + mock_oi = MagicMock() + mock_oi.id = "oi_err1" + mock_oi.status = "error" + mock_oi.results = [] + mock_oi.sample = mock_sample + mock_oi.datasource_item = {} + + mock_client = MagicMock() + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_oi])) + + items = await _fetch_output_items(mock_client, "eval_1", "run_1") + + assert len(items) == 1 + item = items[0] + assert item.is_error + assert item.error_code == "QueryExtractionError" + assert item.error_message == "Query list cannot be empty" + assert len(item.scores) == 0 + + async def test_handles_api_failure_gracefully(self) -> None: + from agent_framework_foundry._foundry_evals import _fetch_output_items + + mock_client = MagicMock() + mock_client.evals.runs.output_items.list = AsyncMock(side_effect=TypeError("API error")) + + items = await _fetch_output_items(mock_client, "eval_1", "run_1") + assert items == [] + + +# --------------------------------------------------------------------------- +# _poll_eval_run — timeout / failed / canceled paths +# --------------------------------------------------------------------------- + + +class TestPollEvalRun: + async def test_timeout_returns_timeout_status(self) -> None: + """Poll timeout returns EvalResults with status='timeout'.""" + from agent_framework_foundry._foundry_evals import _poll_eval_run + + mock_client = MagicMock() + mock_pending = MagicMock() + mock_pending.status = "queued" + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_pending) + + results = await _poll_eval_run(mock_client, "eval_1", "run_1", poll_interval=0.01, timeout=0.05) + assert results.status == "timeout" + assert results.eval_id == "eval_1" + assert results.run_id == "run_1" + + async def test_failed_run_returns_error(self) -> None: + """Failed run returns EvalResults with error message.""" + from agent_framework_foundry._foundry_evals import _poll_eval_run + + mock_client = MagicMock() + mock_failed = MagicMock() + mock_failed.status = "failed" + mock_failed.error = "Model deployment unavailable" + mock_failed.result_counts = None + mock_failed.report_url = None + mock_failed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_failed) + + results = await _poll_eval_run(mock_client, "eval_1", "run_1", poll_interval=0.01, timeout=5.0) + assert results.status == "failed" + assert results.error == "Model deployment unavailable" + assert results.items == [] + + async def test_canceled_run_returns_canceled_status(self) -> None: + """Canceled run returns EvalResults with status='canceled'.""" + from agent_framework_foundry._foundry_evals import _poll_eval_run + + mock_client = MagicMock() + mock_canceled = MagicMock() + mock_canceled.status = "canceled" + mock_canceled.error = None + mock_canceled.result_counts = None + mock_canceled.report_url = None + mock_canceled.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_canceled) + + results = await _poll_eval_run(mock_client, "eval_1", "run_1", poll_interval=0.01, timeout=5.0) + assert results.status == "canceled" + assert results.error is None + assert results.items == [] + + +# --------------------------------------------------------------------------- +# evaluate_traces +# --------------------------------------------------------------------------- + + +class TestEvaluateTraces: + async def test_raises_without_required_args(self) -> None: + """Raises ValueError when no response_ids, trace_ids, or agent_id given.""" + from agent_framework_foundry._foundry_evals import evaluate_traces + + mock_client = MagicMock() + with pytest.raises(ValueError, match="Provide at least one of"): + await evaluate_traces( + client=mock_client, + model="gpt-4o", + ) + + async def test_response_ids_path(self) -> None: + """evaluate_traces with response_ids uses the responses API path.""" + from agent_framework_foundry._foundry_evals import evaluate_traces + + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tr" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tr" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = "https://portal.azure.com/eval/run_tr" + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + mock_output_item = MagicMock() + mock_output_item.id = "oi_resp" + mock_output_item.status = "pass" + mock_output_item.sample = MagicMock(error=None, usage=None, input=[], output=[]) + mock_result = MagicMock(status="pass", score=4) + mock_result.name = "relevance" + mock_output_item.results = [mock_result] + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([mock_output_item])) + + results = await evaluate_traces( + response_ids=["resp_abc", "resp_def"], + client=mock_client, + model="gpt-4o", + ) + assert results.status == "completed" + assert results.eval_id == "eval_tr" + assert len(results.items) == 1 + assert results.items[0].item_id == "oi_resp" + + # Verify the response IDs are in the data source + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "azure_ai_responses" + content = ds["item_generation_params"]["source"]["content"] + assert len(content) == 2 + assert content[0]["item"]["resp_id"] == "resp_abc" + + async def test_trace_ids_path(self) -> None: + """evaluate_traces with trace_ids builds azure_ai_traces data source.""" + from agent_framework_foundry._foundry_evals import evaluate_traces + + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tid" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tid" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=1) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + results = await evaluate_traces( + trace_ids=["trace_1"], + client=mock_client, + model="gpt-4o", + ) + assert results.status == "completed" + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "azure_ai_traces" + assert ds["trace_ids"] == ["trace_1"] + + +# --------------------------------------------------------------------------- +# evaluate_foundry_target +# --------------------------------------------------------------------------- + + +class TestEvaluateFoundryTarget: + async def test_happy_path(self) -> None: + """evaluate_foundry_target creates eval + run and polls to completion.""" + from agent_framework_foundry._foundry_evals import evaluate_foundry_target + + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_tgt" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_tgt" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=2) + mock_completed.report_url = "https://portal.azure.com/eval/run_tgt" + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + results = await evaluate_foundry_target( + target={"type": "azure_ai_agent", "name": "my-agent"}, + test_queries=["Query 1", "Query 2"], + client=mock_client, + model="gpt-4o", + ) + assert results.status == "completed" + assert results.eval_id == "eval_tgt" + assert results.all_passed + + # Verify the target and queries in data source + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "azure_ai_target_completions" + assert ds["target"]["type"] == "azure_ai_agent" + content = ds["source"]["content"] + assert len(content) == 2 + assert content[0]["item"]["query"] == "Query 1" + + +# --------------------------------------------------------------------------- +# r3 review: _extract_result_counts paths +# --------------------------------------------------------------------------- + + +class TestExtractResultCounts: + """Tests for all _extract_result_counts code paths.""" + + def test_typed_counts(self) -> None: + """ResultCounts-like object with all fields.""" + run = MagicMock() + run.result_counts = _rc(passed=3, failed=1) + result = _extract_result_counts(run) + assert result == {"errored": 0, "failed": 1, "passed": 3, "total": 4} + + def test_none_result_counts(self): + run = MagicMock() + run.result_counts = None + assert _extract_result_counts(run) is None + + +# --------------------------------------------------------------------------- +# r3 review: _extract_per_evaluator +# --------------------------------------------------------------------------- + + +class TestExtractPerEvaluator: + """Tests for _extract_per_evaluator with mock data.""" + + def test_with_per_testing_criteria_results(self): + """Parses per_testing_criteria_results into per-evaluator breakdown.""" + + @dataclass + class CriteriaItem: + testing_criteria: str + passed: int + failed: int + + run = MagicMock() + run.per_testing_criteria_results = [ + CriteriaItem("relevance", 4, 1), + CriteriaItem("coherence", 5, 0), + ] + result = _extract_per_evaluator(run) + assert "relevance" in result + assert result["relevance"] == {"passed": 4, "failed": 1} + assert "coherence" in result + assert result["coherence"] == {"passed": 5, "failed": 0} + + def test_with_testing_criteria_attr(self): + """Uses testing_criteria field (the real SDK field name).""" + + @dataclass + class CriteriaItem: + testing_criteria: str + passed: int + failed: int + + run = MagicMock() + run.per_testing_criteria_results = [CriteriaItem("fluency", 3, 2)] + result = _extract_per_evaluator(run) + assert "fluency" in result + assert result["fluency"]["passed"] == 3 + + def test_none_per_testing_criteria(self): + run = MagicMock() + run.per_testing_criteria_results = None + assert _extract_per_evaluator(run) == {} + + +# --------------------------------------------------------------------------- +# r3 review: _resolve_openai_client async check +# --------------------------------------------------------------------------- + + +class TestResolveOpenaiClientAsyncCheck: + """Tests for the async client runtime check.""" + + def test_sync_client_raises(self): + """A sync project_client raises TypeError (not an AsyncOpenAI instance).""" + mock_project = MagicMock() + sync_client = MagicMock() # plain MagicMock, not isinstance(AsyncOpenAI) + mock_project.get_openai_client.return_value = sync_client + + with pytest.raises(TypeError, match="sync client"): + _resolve_openai_client(project_client=mock_project) + + +# --------------------------------------------------------------------------- +# r5 review: evaluator set consistency (replaces import-time asserts) +# --------------------------------------------------------------------------- + + +class TestEvaluatorSetConsistency: + """Verify that _AGENT_EVALUATORS and _TOOL_EVALUATORS are subsets of _BUILTIN_EVALUATORS.""" + + def test_agent_evaluators_subset(self): + from agent_framework_foundry._foundry_evals import _AGENT_EVALUATORS, _BUILTIN_EVALUATORS + + diff = _AGENT_EVALUATORS - set(_BUILTIN_EVALUATORS.values()) + assert not diff, f"_AGENT_EVALUATORS has names not in _BUILTIN_EVALUATORS: {diff}" + + def test_tool_evaluators_subset(self): + from agent_framework_foundry._foundry_evals import _BUILTIN_EVALUATORS, _TOOL_EVALUATORS + + diff = _TOOL_EVALUATORS - set(_BUILTIN_EVALUATORS.values()) + assert not diff, f"_TOOL_EVALUATORS has names not in _BUILTIN_EVALUATORS: {diff}" + + +# --------------------------------------------------------------------------- +# r5 review: evaluate_traces with agent_id only +# --------------------------------------------------------------------------- + + +class TestEvaluateTracesAgentId: + async def test_agent_id_only_path(self) -> None: + """evaluate_traces with agent_id only builds azure_ai_traces data source.""" + from agent_framework_foundry._foundry_evals import evaluate_traces + + mock_client = MagicMock() + + mock_eval = MagicMock() + mock_eval.id = "eval_aid" + mock_client.evals.create = AsyncMock(return_value=mock_eval) + + mock_run = MagicMock() + mock_run.id = "run_aid" + mock_client.evals.runs.create = AsyncMock(return_value=mock_run) + + mock_completed = MagicMock() + mock_completed.status = "completed" + mock_completed.result_counts = _rc(passed=2) + mock_completed.report_url = None + mock_completed.per_testing_criteria_results = None + mock_client.evals.runs.retrieve = AsyncMock(return_value=mock_completed) + + mock_client.evals.runs.output_items.list = AsyncMock(return_value=_AsyncPage([])) + + results = await evaluate_traces( + agent_id="my-agent", + client=mock_client, + model="gpt-4o", + lookback_hours=24, + ) + assert results.status == "completed" + + run_call = mock_client.evals.runs.create.call_args + ds = run_call.kwargs["data_source"] + assert ds["type"] == "azure_ai_traces" + assert ds["agent_id"] == "my-agent" + assert ds["lookback_hours"] == 24 + assert "trace_ids" not in ds + + +# --------------------------------------------------------------------------- +# r5 review: _filter_tool_evaluators raises ValueError +# --------------------------------------------------------------------------- + + +class TestFilterToolEvaluatorsRaises: + def test_all_tool_evaluators_no_tools_raises(self): + """All tool evaluators + no items with tools → ValueError.""" + items = [EvalItem(conversation=[Message("user", ["Hi"]), Message("assistant", ["Hello"])])] + with pytest.raises(ValueError, match="require tool definitions"): + _filter_tool_evaluators(["builtin.tool_call_accuracy", "builtin.tool_selection"], items) + + +# --------------------------------------------------------------------------- +# r5 review: evaluate_foundry_target validates target dict +# --------------------------------------------------------------------------- + + +class TestEvaluateFoundryTargetValidation: + async def test_target_without_type_raises(self) -> None: + """target dict without 'type' key raises ValueError.""" + from agent_framework_foundry._foundry_evals import evaluate_foundry_target + + mock_client = MagicMock() + with pytest.raises(ValueError, match="'type' key"): + await evaluate_foundry_target( + target={"name": "my-agent"}, # missing "type" + test_queries=["Hello"], + client=mock_client, + model="gpt-4o", + ) diff --git a/python/packages/foundry_local/pyproject.toml b/python/packages/foundry_local/pyproject.toml index 0137a1e491..f43d6f2fde 100644 --- a/python/packages/foundry_local/pyproject.toml +++ b/python/packages/foundry_local/pyproject.toml @@ -4,7 +4,7 @@ description = "Foundry Local integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,8 +23,8 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", - "agent-framework-openai>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", + "agent-framework-openai>=1.0.0rc6", "foundry-local-sdk>=0.5.1,<0.5.2", ] diff --git a/python/packages/github_copilot/pyproject.toml b/python/packages/github_copilot/pyproject.toml index 55335c7279..5bcb2d3d8a 100644 --- a/python/packages/github_copilot/pyproject.toml +++ b/python/packages/github_copilot/pyproject.toml @@ -4,7 +4,7 @@ description = "GitHub Copilot integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "github-copilot-sdk>=0.1.31,<0.1.33; python_version >= '3.11'", ] diff --git a/python/packages/lab/pyproject.toml b/python/packages/lab/pyproject.toml index 45a4ae374f..07ae248d0b 100644 --- a/python/packages/lab/pyproject.toml +++ b/python/packages/lab/pyproject.toml @@ -4,7 +4,7 @@ description = "Experimental modules for Microsoft Agent Framework" authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -22,7 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", ] [project.optional-dependencies] diff --git a/python/packages/mem0/pyproject.toml b/python/packages/mem0/pyproject.toml index 9a21e15116..46748866ae 100644 --- a/python/packages/mem0/pyproject.toml +++ b/python/packages/mem0/pyproject.toml @@ -4,7 +4,7 @@ description = "Mem0 integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "mem0ai>=1.0.0,<2", ] diff --git a/python/packages/ollama/pyproject.toml b/python/packages/ollama/pyproject.toml index f8c5cd1b7e..b10ea2706f 100644 --- a/python/packages/ollama/pyproject.toml +++ b/python/packages/ollama/pyproject.toml @@ -4,7 +4,7 @@ description = "Ollama integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://learn.microsoft.com/en-us/agent-framework/" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "ollama>=0.5.3,<0.5.4", ] diff --git a/python/packages/openai/pyproject.toml b/python/packages/openai/pyproject.toml index 6a89f5685c..eb31ca5956 100644 --- a/python/packages/openai/pyproject.toml +++ b/python/packages/openai/pyproject.toml @@ -4,7 +4,7 @@ description = "OpenAI integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0rc5" +version = "1.0.0rc6" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "openai>=1.99.0,<3", "packaging>=24.1,<25", ] diff --git a/python/packages/orchestrations/pyproject.toml b/python/packages/orchestrations/pyproject.toml index fc35d01b05..e519086dbf 100644 --- a/python/packages/orchestrations/pyproject.toml +++ b/python/packages/orchestrations/pyproject.toml @@ -4,7 +4,7 @@ description = "Orchestration patterns for Microsoft Agent Framework. Includes Se authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", ] [tool.uv] diff --git a/python/packages/purview/pyproject.toml b/python/packages/purview/pyproject.toml index 71aef81b9a..183de4570c 100644 --- a/python/packages/purview/pyproject.toml +++ b/python/packages/purview/pyproject.toml @@ -4,7 +4,7 @@ description = "Microsoft Purview (Graph dataSecurityAndGovernance) integration f authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://github.com/microsoft/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -24,7 +24,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "azure-core>=1.30.0,<2", "httpx>=0.27.0,<0.29", ] diff --git a/python/packages/redis/pyproject.toml b/python/packages/redis/pyproject.toml index 3b2d3442ed..9cb248ccf9 100644 --- a/python/packages/redis/pyproject.toml +++ b/python/packages/redis/pyproject.toml @@ -4,7 +4,7 @@ description = "Redis integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0b260319" +version = "1.0.0b260330" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core>=1.0.0rc5", + "agent-framework-core>=1.0.0rc6", "redis>=6.4.0,<7.2.1", "redisvl>=0.11.0,<0.16", "numpy>=2.2.6,<3" diff --git a/python/pyproject.toml b/python/pyproject.toml index c19a6024d0..a554ce97f1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ description = "Microsoft Agent Framework for building AI Agents with Python. Thi authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" requires-python = ">=3.10" -version = "1.0.0rc5" +version = "1.0.0rc6" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" urls.source = "https://github.com/microsoft/agent-framework/tree/main/python" @@ -23,7 +23,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "agent-framework-core[all]==1.0.0rc5", + "agent-framework-core[all]==1.0.0rc6", ] [dependency-groups] diff --git a/python/samples/01-get-started/README.md b/python/samples/01-get-started/README.md index e1bae20b32..7d696ba528 100644 --- a/python/samples/01-get-started/README.md +++ b/python/samples/01-get-started/README.md @@ -9,13 +9,6 @@ concepts of **Agent Framework** one step at a time. pip install agent-framework --pre ``` -Set the required environment variables: - -```bash -export AZURE_AI_PROJECT_ENDPOINT="https://your-project-endpoint" -export AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME="gpt-4o" # optional, defaults to gpt-4o -``` - ## Samples | # | File | What you'll learn | diff --git a/python/samples/02-agents/compaction/compaction_provider.py b/python/samples/02-agents/compaction/compaction_provider.py index d91fa42d7c..23483017c2 100644 --- a/python/samples/02-agents/compaction/compaction_provider.py +++ b/python/samples/02-agents/compaction/compaction_provider.py @@ -70,7 +70,7 @@ async def log_model_input(context: ChatContext, call_next: Any) -> None: async def main() -> None: - client = OpenAIChatClient(model_id="gpt-4o-mini") + client = OpenAIChatClient(model="gpt-4o-mini") # History provider loads/stores conversation messages in session.state. # skip_excluded=True means get_messages() will omit messages that were diff --git a/python/samples/02-agents/context_providers/redis/redis_conversation.py b/python/samples/02-agents/context_providers/redis/redis_conversation.py index ad95b141a1..d3abf86394 100644 --- a/python/samples/02-agents/context_providers/redis/redis_conversation.py +++ b/python/samples/02-agents/context_providers/redis/redis_conversation.py @@ -25,11 +25,11 @@ from agent_framework import Agent from agent_framework.foundry import FoundryChatClient from agent_framework.redis import RedisContextProvider from azure.identity import AzureCliCredential +from dotenv import load_dotenv from redisvl.extensions.cache.embeddings import EmbeddingsCache from redisvl.utils.vectorize import OpenAITextVectorizer -# Copyright (c) Microsoft. All rights reserved. - +load_dotenv() # Default Redis URL for local Redis Stack. # Override via the REDIS_URL environment variable for remote or authenticated instances. diff --git a/python/samples/02-agents/evaluation/evaluate_agent.py b/python/samples/02-agents/evaluation/evaluate_agent.py new file mode 100644 index 0000000000..d9cf952743 --- /dev/null +++ b/python/samples/02-agents/evaluation/evaluate_agent.py @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate an agent with local checks — no API keys needed. + +Demonstrates the simplest evaluation workflow: +1. Define checks using the @evaluator decorator +2. Run evaluate_agent() which calls agent.run() under the covers +3. Assert results in CI or inspect interactively + +Usage: + uv run python samples/02-agents/evaluation/evaluate_agent.py +""" + +import asyncio +import os + +from agent_framework import ( + Agent, + LocalEvaluator, + evaluate_agent, + evaluator, + keyword_check, +) +from agent_framework.foundry import FoundryChatClient +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +# A custom check — parameter names determine what data you receive +@evaluator +def is_helpful(response: str) -> bool: + """Check the response isn't empty or a refusal.""" + refusals = ["i can't", "i'm not able", "i don't know"] + return len(response) > 10 and not any(r in response.lower() for r in refusals) + + +async def main() -> None: + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("FOUNDRY_MODEL", "gpt-4o"), + credential=AzureCliCredential(), + ) + + agent = Agent( + client=client, + name="weather-assistant", + instructions="You are a helpful weather assistant.", + ) + + # Combine built-in and custom checks + local = LocalEvaluator( + keyword_check("weather"), # response must mention "weather" + is_helpful, # custom check + ) + + # evaluate_agent() calls agent.run() for each query, then evaluates + results = await evaluate_agent( + agent=agent, + queries=[ + "What's the weather like in Seattle?", + "Will it rain in London tomorrow?", + "What should I wear for 30°C weather?", + ], + evaluators=local, + ) + + for r in results: + print(f"{r.provider}: {r.passed}/{r.total} passed") + for item in r.items: + print(f" [{item.status}] Q: {item.input_text[:50]} A: {item.output_text[:50]}...") + for score in item.scores: + print(f" {'PASS' if score.passed else 'FAIL'} {score.name}") + + # Use in CI: will raise EvalNotPassedError if any check fails + # results[0].raise_for_status() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/02-agents/evaluation/evaluate_multimodal.py b/python/samples/02-agents/evaluation/evaluate_multimodal.py new file mode 100644 index 0000000000..5d456ef4dc --- /dev/null +++ b/python/samples/02-agents/evaluation/evaluate_multimodal.py @@ -0,0 +1,122 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate multimodal (image) conversations locally. + +Demonstrates that the evaluation pipeline preserves image content: +1. Build EvalItems with image content in conversations +2. Use @evaluator checks that inspect multimodal content +3. Verify images flow through the eval pipeline intact + +Usage: + uv run python samples/02-agents/evaluation/evaluate_multimodal.py +""" + +import asyncio +import base64 + +from agent_framework import ( + Content, + EvalItem, + LocalEvaluator, + Message, + evaluator, +) + + +# -- Custom evaluators that inspect multimodal content -- + + +@evaluator +def has_image_content(conversation: list) -> bool: + """Check that the conversation contains at least one image.""" + return any( + c.type in ("data", "uri") and c.media_type and c.media_type.startswith("image/") + for m in conversation + for c in (m.contents or []) + ) + + +@evaluator +def response_describes_image(response: str) -> bool: + """Check that the assistant response acknowledges the image.""" + image_words = {"image", "picture", "photo", "shows", "depicts", "see"} + return any(word in response.lower() for word in image_words) + + +@evaluator +def image_count(conversation: list) -> float: + """Return the number of images in the conversation as a score.""" + count = sum( + 1 + for m in conversation + for c in (m.contents or []) + if c.type in ("data", "uri") and c.media_type and c.media_type.startswith("image/") + ) + return float(count) + + +# A tiny 1x1 red PNG for demonstration (no external dependencies needed) +_TINY_PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" +) + + +async def main() -> None: + # Build eval items with multimodal content (no agent run needed) + items = [ + # Item 1: User sends an image URL with a question + EvalItem( + conversation=[ + Message( + "user", + [ + Content.from_text("What do you see in this image?"), + Content.from_uri( + "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/300px-PNG_transparency_demonstration_1.png", + media_type="image/png", + ), + ], + ), + Message("assistant", ["The image shows two dice on a transparent background."]), + ] + ), + # Item 2: User sends inline image bytes + EvalItem( + conversation=[ + Message( + "user", + [ + Content.from_text("Describe this picture"), + Content.from_data(data=_TINY_PNG, media_type="image/png"), + ], + ), + Message("assistant", ["I see a small red image — it appears to be a single pixel."]), + ] + ), + # Item 3: Text-only conversation (should fail has_image_content) + EvalItem( + conversation=[ + Message("user", ["Tell me about cats"]), + Message("assistant", ["Cats are wonderful pets."]), + ] + ), + ] + + local = LocalEvaluator( + has_image_content, + response_describes_image, + image_count, + ) + + results = await local.evaluate(items) + + print(f"\n{results.provider}: {results.passed}/{results.total} passed") + for item in results.items: + print(f"\n [{item.status}] Q: {item.input_text[:60]}...") + for score in item.scores: + symbol = "PASS" if score.passed else "FAIL" + print(f" {symbol} {score.name}: {score.score}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/02-agents/evaluation/evaluate_with_expected.py b/python/samples/02-agents/evaluation/evaluate_with_expected.py new file mode 100644 index 0000000000..de44c4e7e9 --- /dev/null +++ b/python/samples/02-agents/evaluation/evaluate_with_expected.py @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate an agent with expected outputs and tool call checks. + +Demonstrates ground-truth comparison and tool usage evaluation: +1. Provide expected outputs alongside queries +2. Use built-in tool_calls_present for tool verification +3. Combine multiple evaluation criteria + +Usage: + uv run python samples/02-agents/evaluation/evaluate_with_expected.py +""" + +import asyncio +import os + +from agent_framework import ( + Agent, + LocalEvaluator, + evaluate_agent, + evaluator, + tool_calls_present, +) +from agent_framework.foundry import FoundryChatClient +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +@evaluator +def response_matches_expected(response: str, expected_output: str) -> float: + """Score based on word overlap with expected output.""" + if not expected_output: + return 1.0 + response_words = set(response.lower().split()) + expected_words = set(expected_output.lower().split()) + return len(response_words & expected_words) / max(len(expected_words), 1) + + +async def main() -> None: + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + agent = Agent( + client=client, + name="math-tutor", + instructions="You are a math tutor. Answer concisely.", + ) + + local = LocalEvaluator( + response_matches_expected, + tool_calls_present, # verifies expected tools were called + ) + + results = await evaluate_agent( + agent=agent, + queries=["What is 2 + 2?", "What is the square root of 144?"], + expected_output=["4", "12"], + evaluators=local, + ) + + for r in results: + print(f"{r.provider}: {r.passed}/{r.total} passed") + for item in r.items: + print(f" [{item.status}] {item.input_text} -> {item.output_text[:80]}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/02-agents/typed_options.py b/python/samples/02-agents/typed_options.py index f33cc21186..9f2862a65a 100644 --- a/python/samples/02-agents/typed_options.py +++ b/python/samples/02-agents/typed_options.py @@ -3,7 +3,7 @@ import asyncio from typing import Literal -from agent_framework import Agent +from agent_framework import Agent, Message from agent_framework.anthropic import AnthropicClient from agent_framework.foundry import FoundryChatClient from agent_framework.openai import OpenAIChatClient, OpenAIChatOptions @@ -40,11 +40,11 @@ async def demo_anthropic_chat_client() -> None: print("\n=== Anthropic ChatClient with TypedDict Options ===\n") # Create Anthropic client - client = AnthropicClient(model="claude-sonnet-4-5-20250929") + client = AnthropicClient(model_id="claude-sonnet-4-5-20250929") # Standard options work great: response = await client.get_response( - "What is the capital of France?", + [Message("user", text="What is the capital of France?")], options={ "temperature": 0.5, "max_tokens": 1000, @@ -62,7 +62,7 @@ async def demo_anthropic_agent() -> None: """Demonstrate Agent with Anthropic client and typed options.""" print("\n=== Agent with Anthropic and Typed Options ===\n") - client = AnthropicClient(model="claude-sonnet-4-5-20250929") + client = AnthropicClient(model_id="claude-sonnet-4-5-20250929") # Create a typed agent for Anthropic - IDE knows Anthropic-specific options! agent = Agent( @@ -119,12 +119,12 @@ async def demo_openai_chat_client_reasoning_models() -> None: print("\n=== OpenAI ChatClient with TypedDict Options ===\n") # Create OpenAI client - client = OpenAIChatClient[OpenAIReasoningChatOptions](model_id="o3") + client = OpenAIChatClient[OpenAIReasoningChatOptions](model="o3") # With specific options, you get full IDE autocomplete! # Try typing `client.get_response("Hello", options={` and see the suggestions response = await client.get_response( - "What is 2 + 2?", + [Message("user", text="What is 2 + 2?")], options={ "max_tokens": 100, "allow_multiple_tool_calls": True, diff --git a/python/samples/03-workflows/README.md b/python/samples/03-workflows/README.md index 4dfdd68157..0e5dbaa9c0 100644 --- a/python/samples/03-workflows/README.md +++ b/python/samples/03-workflows/README.md @@ -172,7 +172,7 @@ Workflow and orchestration samples use `AzureOpenAIResponsesClient` rather than Workflow samples that use `AzureOpenAIResponsesClient` expect: -- `AZURE_AI_PROJECT_ENDPOINT` (Azure AI Foundry Agent Service (V2) project endpoint) -- `AZURE_AI_MODEL_DEPLOYMENT_NAME` (model deployment name) +- `FOUNDRY_PROJECT_ENDPOINT` (Azure AI Foundry Agent Service (V2) project endpoint) +- `FOUNDRY_MODEL` (model deployment name) These values are passed directly into the client constructor via `os.getenv()` in sample code. diff --git a/python/samples/03-workflows/_start-here/step2_agents_in_a_workflow.py b/python/samples/03-workflows/_start-here/step2_agents_in_a_workflow.py index 326bd717ed..51a138429f 100644 --- a/python/samples/03-workflows/_start-here/step2_agents_in_a_workflow.py +++ b/python/samples/03-workflows/_start-here/step2_agents_in_a_workflow.py @@ -24,7 +24,7 @@ how agents can be used in a workflow. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be the deployment name of a model in your Foundry project. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, edges, events, and streaming or non-streaming runs. """ @@ -35,7 +35,7 @@ async def main(): # Create the Azure chat client. AzureCliCredential uses your current az login. client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) writer_agent = Agent( diff --git a/python/samples/03-workflows/_start-here/step3_streaming.py b/python/samples/03-workflows/_start-here/step3_streaming.py index 7d66e1f137..d2c29b56fc 100644 --- a/python/samples/03-workflows/_start-here/step3_streaming.py +++ b/python/samples/03-workflows/_start-here/step3_streaming.py @@ -23,7 +23,7 @@ how agents can be used in a workflow. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be the deployment name of a model in your Foundry project. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, executors, edges, events, and streaming runs. """ @@ -34,7 +34,7 @@ async def main(): # Create the Azure chat client. AzureCliCredential uses your current az login. client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) writer_agent = Agent( diff --git a/python/samples/03-workflows/agents/azure_ai_agents_streaming.py b/python/samples/03-workflows/agents/azure_ai_agents_streaming.py index e54dd1b8f4..9777e26f5a 100644 --- a/python/samples/03-workflows/agents/azure_ai_agents_streaming.py +++ b/python/samples/03-workflows/agents/azure_ai_agents_streaming.py @@ -18,7 +18,7 @@ This sample shows how to create agents backed by Azure OpenAI Responses and use Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- AZURE_AI_MODEL_DEPLOYMENT_NAME must be set to your Azure OpenAI model deployment name. +- FOUNDRY_MODEL must be the deployment name of a model in your Foundry project. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, edges, events, and streaming runs. """ @@ -27,7 +27,7 @@ Prerequisites: async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/agents/azure_ai_agents_with_shared_session.py b/python/samples/03-workflows/agents/azure_ai_agents_with_shared_session.py index bc4167d466..3600c8ce33 100644 --- a/python/samples/03-workflows/agents/azure_ai_agents_with_shared_session.py +++ b/python/samples/03-workflows/agents/azure_ai_agents_with_shared_session.py @@ -39,7 +39,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- AZURE_AI_MODEL_DEPLOYMENT_NAME must be set to your Azure OpenAI model deployment name. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with agents, workflows, and executors in the agent framework. """ @@ -60,7 +60,7 @@ async def intercept_agent_response( async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/agents/azure_chat_agents_and_executor.py b/python/samples/03-workflows/agents/azure_chat_agents_and_executor.py index 9655781522..d658abe3b1 100644 --- a/python/samples/03-workflows/agents/azure_chat_agents_and_executor.py +++ b/python/samples/03-workflows/agents/azure_chat_agents_and_executor.py @@ -37,7 +37,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Run `az login` before executing. """ @@ -104,7 +104,7 @@ async def main() -> None: research_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="research_agent", @@ -116,7 +116,7 @@ async def main() -> None: final_editor_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="final_editor_agent", diff --git a/python/samples/03-workflows/agents/azure_chat_agents_streaming.py b/python/samples/03-workflows/agents/azure_chat_agents_streaming.py index a7fe68f27f..39a7777663 100644 --- a/python/samples/03-workflows/agents/azure_chat_agents_streaming.py +++ b/python/samples/03-workflows/agents/azure_chat_agents_streaming.py @@ -18,7 +18,7 @@ This sample shows how to create AzureOpenAI Chat Agents and use them in a workfl Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, edges, events, and streaming runs. """ @@ -29,7 +29,7 @@ async def main(): # Create the agents _writer_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) writer_agent = Agent( @@ -42,7 +42,7 @@ async def main(): _reviewer_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) reviewer_agent = Agent( diff --git a/python/samples/03-workflows/agents/azure_chat_agents_tool_calls_with_feedback.py b/python/samples/03-workflows/agents/azure_chat_agents_tool_calls_with_feedback.py index 5c1d3027b0..1034039a34 100644 --- a/python/samples/03-workflows/agents/azure_chat_agents_tool_calls_with_feedback.py +++ b/python/samples/03-workflows/agents/azure_chat_agents_tool_calls_with_feedback.py @@ -49,7 +49,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Run `az login` before executing. """ @@ -122,11 +122,7 @@ class Coordinator(Executor): # Writer agent response; request human feedback. # Preserve the full conversation so the final editor # can see tool traces and the initial prompt. - conversation: list[Message] - if draft.full_conversation is not None: - conversation = list(draft.full_conversation) - else: - conversation = list(draft.agent_response.messages) + conversation = list(draft.full_conversation) draft_text = draft.agent_response.text.strip() if not draft_text: draft_text = "No draft text was produced." @@ -178,7 +174,7 @@ def create_writer_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="writer_agent", @@ -188,7 +184,9 @@ def create_writer_agent() -> Agent: "produce a 3-sentence draft." ), tools=[fetch_product_brief, get_brand_voice_profile], - tool_choice="required", + default_options={ + "tool_choice": "required", + }, ) @@ -197,7 +195,7 @@ def create_final_editor_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="final_editor_agent", diff --git a/python/samples/03-workflows/agents/concurrent_workflow_as_agent.py b/python/samples/03-workflows/agents/concurrent_workflow_as_agent.py index acee104c86..42538e31a4 100644 --- a/python/samples/03-workflows/agents/concurrent_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/concurrent_workflow_as_agent.py @@ -25,7 +25,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI access configured for FoundryChatClient (use az login + env vars) +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Familiarity with Workflow events (WorkflowEvent with type "output") """ @@ -34,7 +34,7 @@ async def main() -> None: # 1) Create three domain agents using FoundryChatClient client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -69,7 +69,7 @@ async def main() -> None: workflow = ConcurrentBuilder(participants=[researcher, marketer, legal]).build() # 3) Expose the concurrent workflow as an agent for easy reuse - agent = Agent(client=workflow, name="ConcurrentWorkflowAgent") + agent = workflow.as_agent() prompt = "We are launching a new budget-friendly electric bike for urban commuters." agent_response = await agent.run(prompt) diff --git a/python/samples/03-workflows/agents/custom_agent_executors.py b/python/samples/03-workflows/agents/custom_agent_executors.py index 95d1d8dca8..af12db1cf5 100644 --- a/python/samples/03-workflows/agents/custom_agent_executors.py +++ b/python/samples/03-workflows/agents/custom_agent_executors.py @@ -33,7 +33,7 @@ Note: When an agent is passed to a workflow, the workflow wraps the agent in a m Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, executors, edges, events, and streaming or non streaming runs. """ @@ -54,7 +54,7 @@ class Writer(Executor): self.agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -101,7 +101,7 @@ class Reviewer(Executor): self.agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py b/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py index 3c03a2fb57..b503f7574f 100644 --- a/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py @@ -32,7 +32,7 @@ async def main() -> None: instructions="Gather concise facts that help a teammate answer the question.", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -43,14 +43,14 @@ async def main() -> None: instructions="Compose clear and structured answers using any notes provided.", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) _orch_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -72,7 +72,7 @@ async def main() -> None: print(f"Input: {task}\n") try: - workflow_agent = Agent(client=workflow, name="GroupChatWorkflowAgent") + workflow_agent = workflow.as_agent() agent_result = await workflow_agent.run(task) if agent_result.messages: diff --git a/python/samples/03-workflows/agents/handoff_workflow_as_agent.py b/python/samples/03-workflows/agents/handoff_workflow_as_agent.py index 6c56d3d66b..c7d06b535a 100644 --- a/python/samples/03-workflows/agents/handoff_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/handoff_workflow_as_agent.py @@ -31,7 +31,7 @@ them to transfer control to each other based on the conversation context. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. - `az login` (Azure CLI authentication) - - Environment variables configured for FoundryChatClient (AZURE_AI_MODEL_DEPLOYMENT_NAME) + - Environment variables configured for FoundryChatClient (FOUNDRY_MODEL) Key Concepts: - Auto-registered handoff tools: HandoffBuilder automatically creates handoff tools @@ -159,7 +159,7 @@ async def main() -> None: # Initialize the Azure OpenAI chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -174,21 +174,20 @@ async def main() -> None: # Without this, the default behavior continues requesting user input until max_turns # is reached. Here we use a custom condition that checks if the conversation has ended # naturally (when one of the agents says something like "you're welcome"). - agent = Agent( - client=( - HandoffBuilder( - name="customer_support_handoff", - participants=[triage, refund, order, support], - # Custom termination: Check if one of the agents has provided a closing message. - # This looks for the last message containing "welcome", which indicates the - # conversation has concluded naturally. - termination_condition=lambda conversation: ( - len(conversation) > 0 and "welcome" in conversation[-1].text.lower() - ), - ) - .with_start_agent(triage) - .build() - ), + agent = ( + HandoffBuilder( + name="customer_support_handoff", + participants=[triage, refund, order, support], + # Custom termination: Check if one of the agents has provided a closing message. + # This looks for the last message containing "welcome", which indicates the + # conversation has concluded naturally. + termination_condition=lambda conversation: ( + len(conversation) > 0 and "welcome" in conversation[-1].text.lower() + ), + ) + .with_start_agent(triage) + .build() + .as_agent() ) # Scripted user responses for reproducible demo @@ -226,7 +225,7 @@ async def main() -> None: responses = {req_id: HandoffAgentUserRequest.create_response(user_response) for req_id in pending_requests} function_results = [ - Content.from_function_result(call_id=req_id, result=response) for req_id, response in responses.items() + Content("function_result", call_id=req_id, result=response) for req_id, response in responses.items() ] response = await agent.run(Message("tool", function_results)) pending_requests = handle_response_and_requests(response) diff --git a/python/samples/03-workflows/agents/magentic_workflow_as_agent.py b/python/samples/03-workflows/agents/magentic_workflow_as_agent.py index 7ce8c496df..6cc91a9dcd 100644 --- a/python/samples/03-workflows/agents/magentic_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/magentic_workflow_as_agent.py @@ -23,7 +23,7 @@ like any other agent while still emitting callback telemetry. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI credentials configured for `FoundryChatClient` and `FoundryChatClient`. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -37,7 +37,7 @@ async def main() -> None: # This agent requires the gpt-4o-search-preview model to perform web searches. client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -45,7 +45,7 @@ async def main() -> None: # Create code interpreter tool using instance method coder_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) code_interpreter_tool = coder_client.get_code_interpreter_tool() @@ -65,7 +65,7 @@ async def main() -> None: instructions="You coordinate a team to complete complex tasks efficiently.", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -98,7 +98,7 @@ async def main() -> None: try: # Wrap the workflow as an agent for composition scenarios print("\nWrapping workflow as an agent and running...") - workflow_agent = Agent(client=workflow, name="MagenticWorkflowAgent") + workflow_agent = workflow.as_agent() last_response_id: str | None = None async for update in workflow_agent.run(task, stream=True): diff --git a/python/samples/03-workflows/agents/sequential_workflow_as_agent.py b/python/samples/03-workflows/agents/sequential_workflow_as_agent.py index dcc4d9fad9..52de975173 100644 --- a/python/samples/03-workflows/agents/sequential_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/sequential_workflow_as_agent.py @@ -27,7 +27,7 @@ Note on internal adapters: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI access configured for FoundryChatClient (use az login + env vars) +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -35,7 +35,7 @@ async def main() -> None: # 1) Create agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -55,7 +55,7 @@ async def main() -> None: workflow = SequentialBuilder(participants=[writer, reviewer]).build() # 3) Treat the workflow itself as an agent for follow-up invocations - agent = Agent(client=workflow, name="SequentialWorkflowAgent") + agent = workflow.as_agent() prompt = "Write a tagline for a budget-friendly eBike." agent_response = await agent.run(prompt) diff --git a/python/samples/03-workflows/agents/workflow_as_agent_human_in_the_loop.py b/python/samples/03-workflows/agents/workflow_as_agent_human_in_the_loop.py index d5a5e1f4e2..3b8ccc0faa 100644 --- a/python/samples/03-workflows/agents/workflow_as_agent_human_in_the_loop.py +++ b/python/samples/03-workflows/agents/workflow_as_agent_human_in_the_loop.py @@ -8,7 +8,6 @@ from dataclasses import dataclass from pathlib import Path from typing import Any -from agent_framework import Agent from agent_framework.foundry import FoundryChatClient from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -50,7 +49,7 @@ to the Worker. The workflow completes when idle. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI account configured and accessible for FoundryChatClient. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Familiarity with WorkflowBuilder, Executor, and WorkflowContext from agent_framework. - Understanding of request-response message handling in executors. - (Optional) Review of reflection and escalation patterns, such as those in @@ -113,14 +112,14 @@ async def main() -> None: id="worker", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) reviewer = ReviewerWithHumanInTheLoop(worker_id="worker") - agent = Agent( - client=(WorkflowBuilder(start_executor=worker).add_edge(worker, reviewer).add_edge(reviewer, worker).build()), + agent = ( + WorkflowBuilder(start_executor=worker).add_edge(worker, reviewer).add_edge(reviewer, worker).build().as_agent() ) print("Running workflow agent with user query...") @@ -165,7 +164,8 @@ async def main() -> None: human_response = ReviewResponse(request_id=request_id, feedback="", approved=True) # Create the function call result object to send back to the agent. - human_review_function_result = Content.from_function_result( + human_review_function_result = Content( + "function_result", call_id=human_review_function_call.call_id, # type: ignore result=human_response, ) diff --git a/python/samples/03-workflows/agents/workflow_as_agent_kwargs.py b/python/samples/03-workflows/agents/workflow_as_agent_kwargs.py index 1b8a875773..7c30106cc8 100644 --- a/python/samples/03-workflows/agents/workflow_as_agent_kwargs.py +++ b/python/samples/03-workflows/agents/workflow_as_agent_kwargs.py @@ -35,7 +35,7 @@ When to use Agent(client=workflow,): Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -89,7 +89,7 @@ async def main() -> None: # Create chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -109,7 +109,7 @@ async def main() -> None: workflow = SequentialBuilder(participants=[agent]).build() # Expose the workflow as an agent Agent(client=using,) - workflow_agent = Agent(client=workflow, name="WorkflowAgent") + workflow_agent = workflow.as_agent() # Define custom context that will flow to tools via kwargs custom_data = { diff --git a/python/samples/03-workflows/agents/workflow_as_agent_reflection_pattern.py b/python/samples/03-workflows/agents/workflow_as_agent_reflection_pattern.py index 2d37778b84..d2d02c9581 100644 --- a/python/samples/03-workflows/agents/workflow_as_agent_reflection_pattern.py +++ b/python/samples/03-workflows/agents/workflow_as_agent_reflection_pattern.py @@ -6,7 +6,6 @@ from dataclasses import dataclass from uuid import uuid4 from agent_framework import ( - Agent, AgentResponse, Executor, Message, @@ -41,7 +40,7 @@ Key Concepts Demonstrated: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI account configured and accessible for FoundryChatClient. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Familiarity with WorkflowBuilder, Executor, WorkflowContext, and event handling. - Understanding of how agent messages are generated, reviewed, and re-submitted. """ @@ -198,7 +197,7 @@ async def main() -> None: id="worker", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -206,13 +205,13 @@ async def main() -> None: id="reviewer", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) - agent = Agent( - client=(WorkflowBuilder(start_executor=worker).add_edge(worker, reviewer).add_edge(reviewer, worker).build()), + agent = ( + WorkflowBuilder(start_executor=worker).add_edge(worker, reviewer).add_edge(reviewer, worker).build().as_agent() ) print("Running workflow agent with user query...") diff --git a/python/samples/03-workflows/agents/workflow_as_agent_with_session.py b/python/samples/03-workflows/agents/workflow_as_agent_with_session.py index 469568f000..7d65b36492 100644 --- a/python/samples/03-workflows/agents/workflow_as_agent_with_session.py +++ b/python/samples/03-workflows/agents/workflow_as_agent_with_session.py @@ -38,7 +38,7 @@ Use cases: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured for FoundryChatClient +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -46,7 +46,7 @@ async def main() -> None: # Create a chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -72,7 +72,7 @@ async def main() -> None: workflow = SequentialBuilder(participants=[assistant, summarizer]).build() # Wrap the workflow as an agent - agent = Agent(client=workflow, name="ConversationalWorkflowAgent") + agent = workflow.as_agent() # Create a session to maintain history session = agent.create_session() @@ -133,7 +133,7 @@ async def demonstrate_session_serialization() -> None: """ client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -144,7 +144,7 @@ async def demonstrate_session_serialization() -> None: ) workflow = SequentialBuilder(participants=[memory_assistant]).build() - agent = Agent(client=workflow, name="MemoryWorkflowAgent") + agent = workflow.as_agent() # Create initial session and have a conversation session = agent.create_session() diff --git a/python/samples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop.py b/python/samples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop.py index 86c07cfa47..547bb1c6d3 100644 --- a/python/samples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop.py +++ b/python/samples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop.py @@ -182,7 +182,7 @@ def create_workflow(checkpoint_storage: FileCheckpointStorage) -> Workflow: writer_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions="Write concise, warm release notes that sound human and helpful.", diff --git a/python/samples/03-workflows/checkpoint/workflow_as_agent_checkpoint.py b/python/samples/03-workflows/checkpoint/workflow_as_agent_checkpoint.py index bb50e9f0a6..95f7d54b5b 100644 --- a/python/samples/03-workflows/checkpoint/workflow_as_agent_checkpoint.py +++ b/python/samples/03-workflows/checkpoint/workflow_as_agent_checkpoint.py @@ -21,7 +21,7 @@ Key concepts: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured for FoundryChatClient +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ import asyncio @@ -50,7 +50,7 @@ async def basic_checkpointing() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -67,7 +67,7 @@ async def basic_checkpointing() -> None: ) workflow = SequentialBuilder(participants=[assistant, reviewer]).build() - agent = Agent(client=workflow, name="CheckpointedAgent") + agent = workflow.as_agent() # Create checkpoint storage checkpoint_storage = InMemoryCheckpointStorage() @@ -97,7 +97,7 @@ async def checkpointing_with_thread() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -108,7 +108,7 @@ async def checkpointing_with_thread() -> None: ) workflow = SequentialBuilder(participants=[assistant]).build() - agent = Agent(client=workflow, name="MemoryAgent") + agent = workflow.as_agent() # Create both session (for conversation) and checkpoint storage (for workflow state) session = agent.create_session() @@ -145,7 +145,7 @@ async def streaming_with_checkpoints() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) @@ -156,7 +156,7 @@ async def streaming_with_checkpoints() -> None: ) workflow = SequentialBuilder(participants=[assistant]).build() - agent = Agent(client=workflow, name="StreamingCheckpointAgent") + agent = workflow.as_agent() checkpoint_storage = InMemoryCheckpointStorage() diff --git a/python/samples/03-workflows/composition/sub_workflow_kwargs.py b/python/samples/03-workflows/composition/sub_workflow_kwargs.py index 4404cf7b13..d3991e0218 100644 --- a/python/samples/03-workflows/composition/sub_workflow_kwargs.py +++ b/python/samples/03-workflows/composition/sub_workflow_kwargs.py @@ -34,7 +34,7 @@ Key Concepts: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -84,7 +84,7 @@ async def main() -> None: # Create chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/control-flow/edge_condition.py b/python/samples/03-workflows/control-flow/edge_condition.py index 89999ecaa3..e7969b21aa 100644 --- a/python/samples/03-workflows/control-flow/edge_condition.py +++ b/python/samples/03-workflows/control-flow/edge_condition.py @@ -139,7 +139,7 @@ def create_spam_detector_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -158,7 +158,7 @@ def create_email_assistant_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/03-workflows/control-flow/multi_selection_edge_group.py b/python/samples/03-workflows/control-flow/multi_selection_edge_group.py index a5a96ad14f..36c7180642 100644 --- a/python/samples/03-workflows/control-flow/multi_selection_edge_group.py +++ b/python/samples/03-workflows/control-flow/multi_selection_edge_group.py @@ -191,7 +191,7 @@ def create_email_analysis_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -209,7 +209,7 @@ def create_email_assistant_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=("You are an email assistant that helps users draft responses to emails with professionalism."), @@ -223,7 +223,7 @@ def create_email_summary_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=("You are an assistant that helps users summarize emails."), diff --git a/python/samples/03-workflows/control-flow/simple_loop.py b/python/samples/03-workflows/control-flow/simple_loop.py index 3adc75625e..2f0734401d 100644 --- a/python/samples/03-workflows/control-flow/simple_loop.py +++ b/python/samples/03-workflows/control-flow/simple_loop.py @@ -33,7 +33,7 @@ What it does: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure AI/ Azure OpenAI for `FoundryChatClient` agent. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via `azure-identity` — uses `AzureCliCredential()` (run `az login`). """ @@ -126,7 +126,7 @@ def create_judge_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=("You strictly respond with one of: MATCHED, ABOVE, BELOW based on the given target and guess."), diff --git a/python/samples/03-workflows/control-flow/switch_case_edge_group.py b/python/samples/03-workflows/control-flow/switch_case_edge_group.py index b7c1ece95e..9036081a4d 100644 --- a/python/samples/03-workflows/control-flow/switch_case_edge_group.py +++ b/python/samples/03-workflows/control-flow/switch_case_edge_group.py @@ -162,7 +162,7 @@ def create_spam_detection_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -181,7 +181,7 @@ def create_email_assistant_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=("You are an email assistant that helps users draft responses to emails with professionalism."), diff --git a/python/samples/03-workflows/declarative/agent_to_function_tool/main.py b/python/samples/03-workflows/declarative/agent_to_function_tool/main.py index 8f931d4bf5..54e393ee14 100644 --- a/python/samples/03-workflows/declarative/agent_to_function_tool/main.py +++ b/python/samples/03-workflows/declarative/agent_to_function_tool/main.py @@ -204,7 +204,7 @@ async def main(): # Create Azure OpenAI Responses client chat_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/declarative/customer_support/main.py b/python/samples/03-workflows/declarative/customer_support/main.py index 03bd68cff5..374825871d 100644 --- a/python/samples/03-workflows/declarative/customer_support/main.py +++ b/python/samples/03-workflows/declarative/customer_support/main.py @@ -173,7 +173,7 @@ async def main() -> None: project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], # This sample has been tested only on `gpt-5.1` and may not work as intended on other models # This sample is known to fail on `gpt-5-mini` reasoning input (GH issue #4059) - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/declarative/deep_research/main.py b/python/samples/03-workflows/declarative/deep_research/main.py index d6dbb4d65d..49e9b86b8a 100644 --- a/python/samples/03-workflows/declarative/deep_research/main.py +++ b/python/samples/03-workflows/declarative/deep_research/main.py @@ -29,10 +29,11 @@ from agent_framework import Agent from agent_framework.declarative import WorkflowFactory from agent_framework.foundry import FoundryChatClient from azure.identity import AzureCliCredential +from dotenv import load_dotenv from pydantic import BaseModel, Field -# Copyright (c) Microsoft. All rights reserved. - +# Load environment variables from .env file +load_dotenv() # Agent Instructions RESEARCH_INSTRUCTIONS = """In order to help begin addressing the user request, please answer the following pre-survey to the best of your ability. @@ -126,7 +127,7 @@ async def main() -> None: # Create Azure OpenAI client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/declarative/function_tools/main.py b/python/samples/03-workflows/declarative/function_tools/main.py index b8f4ec44f9..8ee4b08e9d 100644 --- a/python/samples/03-workflows/declarative/function_tools/main.py +++ b/python/samples/03-workflows/declarative/function_tools/main.py @@ -71,7 +71,7 @@ async def main(): # Create agent with tools client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) menu_agent = Agent( diff --git a/python/samples/03-workflows/declarative/marketing/main.py b/python/samples/03-workflows/declarative/marketing/main.py index 66a121e5b1..17a7eaacfe 100644 --- a/python/samples/03-workflows/declarative/marketing/main.py +++ b/python/samples/03-workflows/declarative/marketing/main.py @@ -56,7 +56,7 @@ async def main() -> None: """Run the marketing workflow with real Azure AI agents.""" client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/declarative/student_teacher/main.py b/python/samples/03-workflows/declarative/student_teacher/main.py index 415625a300..f9c47fd151 100644 --- a/python/samples/03-workflows/declarative/student_teacher/main.py +++ b/python/samples/03-workflows/declarative/student_teacher/main.py @@ -16,7 +16,7 @@ Prerequisites: - Azure OpenAI deployment with chat completion capability - Environment variables: FOUNDRY_PROJECT_ENDPOINT: Your Azure AI Foundry Agent Service (V2) project endpoint - AZURE_AI_MODEL_DEPLOYMENT_NAME: Your model deployment name + FOUNDRY_MODEL: Your model deployment name """ import asyncio @@ -27,8 +27,10 @@ from agent_framework import Agent from agent_framework.declarative import WorkflowFactory from agent_framework.foundry import FoundryChatClient from azure.identity import AzureCliCredential +from dotenv.main import load_dotenv -# Copyright (c) Microsoft. All rights reserved. +# Load environment variables from .env file +load_dotenv() STUDENT_INSTRUCTIONS = """You are a curious math student working on understanding mathematical concepts. @@ -58,7 +60,7 @@ async def main() -> None: # Create chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/evaluation/evaluate_workflow.py b/python/samples/03-workflows/evaluation/evaluate_workflow.py new file mode 100644 index 0000000000..c0850272bf --- /dev/null +++ b/python/samples/03-workflows/evaluation/evaluate_workflow.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate a multi-agent workflow with per-agent breakdown. + +Demonstrates workflow evaluation: +1. Build a simple two-agent workflow +2. Run evaluate_workflow() which runs the workflow and evaluates each agent +3. Inspect per-agent results in sub_results + +Usage: + uv run python samples/03-workflows/evaluation/evaluate_workflow.py +""" + +import asyncio +import os + +from agent_framework import ( + Agent, + LocalEvaluator, + WorkflowBuilder, + evaluate_workflow, + evaluator, + keyword_check, +) +from agent_framework.foundry import FoundryChatClient +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +@evaluator +def is_nonempty(response: str) -> bool: + """Check the agent produced a non-trivial response.""" + return len(response.strip()) > 5 + + +async def main() -> None: + # Build a simple planner -> executor workflow + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("FOUNDRY_MODEL", "gpt-4o"), + credential=AzureCliCredential(), + ) + planner = Agent(client=client, name="planner", instructions="You plan trips. Output a bullet-point plan.") + executor_agent = Agent( + client=client, name="executor", instructions="You execute travel plans. Book the items listed." + ) + + workflow = WorkflowBuilder(start_executor=planner).add_edge(planner, executor_agent).build() + + # Evaluate with per-agent breakdown + local = LocalEvaluator(is_nonempty, keyword_check("plan", "trip")) + + results = await evaluate_workflow( + workflow=workflow, + queries=["Plan a weekend trip to Paris"], + evaluators=local, + ) + + for r in results: + print(f"{r.provider}: {r.passed}/{r.total} passed (overall)") + for agent_name, sub in r.sub_results.items(): + error = f" (error: {sub.error})" if sub.error else "" + print(f" {agent_name}: {sub.passed}/{sub.total} {error}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/03-workflows/human-in-the-loop/agents_with_HITL.py b/python/samples/03-workflows/human-in-the-loop/agents_with_HITL.py index 42e58083c0..4d45047139 100644 --- a/python/samples/03-workflows/human-in-the-loop/agents_with_HITL.py +++ b/python/samples/03-workflows/human-in-the-loop/agents_with_HITL.py @@ -44,7 +44,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Run `az login` before executing. """ @@ -78,11 +78,7 @@ class Coordinator(Executor): # Writer agent response; request human feedback. # Preserve the full conversation so that the final editor has context. - conversation: list[Message] - if draft.full_conversation is not None: - conversation = list(draft.full_conversation) - else: - conversation = list(draft.agent_response.messages) + conversation = list(draft.full_conversation) prompt = ( "Review the draft from the writer and provide a short directional note " @@ -172,18 +168,20 @@ async def main() -> None: writer_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="writer_agent", instructions=("You are a marketing writer."), - tool_choice="required", + default_options={ + "tool_choice": "required", + }, ) final_editor_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="final_editor_agent", diff --git a/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py b/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py index 85850e78ce..adb1fff4d5 100644 --- a/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py +++ b/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py @@ -53,7 +53,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure AI Agent Service configured, along with the required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, edges, events, request_info events (type='request_info'), and streaming runs. """ @@ -228,7 +228,7 @@ async def main() -> None: email_writer_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="EmailWriter", diff --git a/python/samples/03-workflows/human-in-the-loop/agents_with_declaration_only_tools.py b/python/samples/03-workflows/human-in-the-loop/agents_with_declaration_only_tools.py index 1ab0ea81e7..0c55fb2fc2 100644 --- a/python/samples/03-workflows/human-in-the-loop/agents_with_declaration_only_tools.py +++ b/python/samples/03-workflows/human-in-the-loop/agents_with_declaration_only_tools.py @@ -53,7 +53,7 @@ get_user_location = FunctionTool( async def main() -> None: _client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) agent = Agent( diff --git a/python/samples/03-workflows/human-in-the-loop/concurrent_request_info.py b/python/samples/03-workflows/human-in-the-loop/concurrent_request_info.py index ea5717f337..96dd8ccc52 100644 --- a/python/samples/03-workflows/human-in-the-loop/concurrent_request_info.py +++ b/python/samples/03-workflows/human-in-the-loop/concurrent_request_info.py @@ -18,7 +18,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity (run az login before executing) """ @@ -151,7 +151,7 @@ async def main() -> None: global _chat_client _chat_client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/human-in-the-loop/group_chat_request_info.py b/python/samples/03-workflows/human-in-the-loop/group_chat_request_info.py index c5364d8d47..ff108002ca 100644 --- a/python/samples/03-workflows/human-in-the-loop/group_chat_request_info.py +++ b/python/samples/03-workflows/human-in-the-loop/group_chat_request_info.py @@ -19,7 +19,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity (run az login before executing) """ @@ -99,7 +99,7 @@ async def process_event_stream(stream: AsyncIterable[WorkflowEvent]) -> dict[str async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/human-in-the-loop/guessing_game_with_human_input.py b/python/samples/03-workflows/human-in-the-loop/guessing_game_with_human_input.py index 801d95c8fa..14f6fa2cb0 100644 --- a/python/samples/03-workflows/human-in-the-loop/guessing_game_with_human_input.py +++ b/python/samples/03-workflows/human-in-the-loop/guessing_game_with_human_input.py @@ -44,7 +44,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Basic familiarity with WorkflowBuilder, executors, edges, events, and streaming runs. """ @@ -200,7 +200,7 @@ async def main() -> None: guessing_agent = Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), name="GuessingAgent", diff --git a/python/samples/03-workflows/human-in-the-loop/sequential_request_info.py b/python/samples/03-workflows/human-in-the-loop/sequential_request_info.py index d5294afe01..3b9d1f0e33 100644 --- a/python/samples/03-workflows/human-in-the-loop/sequential_request_info.py +++ b/python/samples/03-workflows/human-in-the-loop/sequential_request_info.py @@ -18,7 +18,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity (run az login before executing) """ @@ -96,7 +96,7 @@ async def process_event_stream(stream: AsyncIterable[WorkflowEvent]) -> dict[str async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/concurrent_agents.py b/python/samples/03-workflows/orchestrations/concurrent_agents.py index 74e53f3970..7e90ff1bda 100644 --- a/python/samples/03-workflows/orchestrations/concurrent_agents.py +++ b/python/samples/03-workflows/orchestrations/concurrent_agents.py @@ -28,7 +28,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Familiarity with Workflow events (WorkflowEvent) """ @@ -38,7 +38,7 @@ async def main() -> None: # 1) Create three domain agents using FoundryChatClient client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/concurrent_custom_agent_executors.py b/python/samples/03-workflows/orchestrations/concurrent_custom_agent_executors.py index 968247f850..4ebdf4ceca 100644 --- a/python/samples/03-workflows/orchestrations/concurrent_custom_agent_executors.py +++ b/python/samples/03-workflows/orchestrations/concurrent_custom_agent_executors.py @@ -38,7 +38,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -109,7 +109,7 @@ class LegalExec(Executor): async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/concurrent_custom_aggregator.py b/python/samples/03-workflows/orchestrations/concurrent_custom_aggregator.py index ad3c849afe..74afd5df85 100644 --- a/python/samples/03-workflows/orchestrations/concurrent_custom_aggregator.py +++ b/python/samples/03-workflows/orchestrations/concurrent_custom_aggregator.py @@ -30,7 +30,7 @@ Demonstrates: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -38,7 +38,7 @@ Prerequisites: async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py b/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py index c1929f8fb1..dea82a4352 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py +++ b/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py @@ -27,7 +27,7 @@ What it does: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -45,7 +45,7 @@ async def main() -> None: # Create a Responses client using Azure OpenAI and Azure CLI credentials for all agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py b/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py index 902bd271c6..867bbd7bc3 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py +++ b/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py @@ -40,7 +40,7 @@ Participants represent: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -51,7 +51,7 @@ load_dotenv() def _get_chat_client() -> FoundryChatClient: return FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py b/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py index 99d7e1a963..2fceaa98d0 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py +++ b/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py @@ -26,7 +26,7 @@ What it does: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -42,7 +42,7 @@ async def main() -> None: # Create a Responses client using Azure OpenAI and Azure CLI credentials for all agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/handoff_autonomous.py b/python/samples/03-workflows/orchestrations/handoff_autonomous.py index 7fcb8842f2..355a782f9d 100644 --- a/python/samples/03-workflows/orchestrations/handoff_autonomous.py +++ b/python/samples/03-workflows/orchestrations/handoff_autonomous.py @@ -84,7 +84,7 @@ async def main() -> None: """Run an autonomous handoff workflow with specialist iteration enabled.""" client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) coordinator, research_agent, summary_agent = create_agents(client) diff --git a/python/samples/03-workflows/orchestrations/handoff_simple.py b/python/samples/03-workflows/orchestrations/handoff_simple.py index d288b51c0d..b804c5e63a 100644 --- a/python/samples/03-workflows/orchestrations/handoff_simple.py +++ b/python/samples/03-workflows/orchestrations/handoff_simple.py @@ -27,7 +27,7 @@ them to transfer control to each other based on the conversation context. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. - - Azure OpenAI configured for FoundryChatClient with required environment variables. + - FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run `az login` before executing the sample. Key Concepts: @@ -201,7 +201,7 @@ async def main() -> None: # Initialize the Azure OpenAI Responses client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/handoff_with_code_interpreter_file.py b/python/samples/03-workflows/orchestrations/handoff_with_code_interpreter_file.py index d6de4efcb5..ced94109a2 100644 --- a/python/samples/03-workflows/orchestrations/handoff_with_code_interpreter_file.py +++ b/python/samples/03-workflows/orchestrations/handoff_with_code_interpreter_file.py @@ -13,8 +13,8 @@ HandoffBuilder workflows can be properly retrieved. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. + - FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - `az login` (Azure CLI authentication) - - AZURE_AI_MODEL_DEPLOYMENT_NAME """ import asyncio @@ -93,7 +93,7 @@ async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/handoff_with_tool_approval_checkpoint_resume.py b/python/samples/03-workflows/orchestrations/handoff_with_tool_approval_checkpoint_resume.py index e1e01c0415..4b65561532 100644 --- a/python/samples/03-workflows/orchestrations/handoff_with_tool_approval_checkpoint_resume.py +++ b/python/samples/03-workflows/orchestrations/handoff_with_tool_approval_checkpoint_resume.py @@ -46,8 +46,8 @@ Pattern: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Azure CLI authentication (az login). -- Environment variables configured for FoundryChatClient. """ CHECKPOINT_DIR = Path(__file__).parent / "tmp" / "handoff_checkpoints" @@ -102,7 +102,7 @@ def create_workflow(checkpoint_storage: FileCheckpointStorage) -> Workflow: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) triage, refund, order = create_agents(client) diff --git a/python/samples/03-workflows/orchestrations/magentic.py b/python/samples/03-workflows/orchestrations/magentic.py index 07ca80a61d..f7a472049c 100644 --- a/python/samples/03-workflows/orchestrations/magentic.py +++ b/python/samples/03-workflows/orchestrations/magentic.py @@ -43,7 +43,7 @@ events, and prints the final answer. The workflow completes when idle. Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -54,7 +54,7 @@ load_dotenv() async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/magentic_checkpoint.py b/python/samples/03-workflows/orchestrations/magentic_checkpoint.py index fba3da09d1..df606a5a43 100644 --- a/python/samples/03-workflows/orchestrations/magentic_checkpoint.py +++ b/python/samples/03-workflows/orchestrations/magentic_checkpoint.py @@ -40,7 +40,7 @@ Concepts highlighted here: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -66,7 +66,7 @@ def build_workflow(checkpoint_storage: FileCheckpointStorage): instructions=("You are the research lead. Gather crisp bullet points the team should know."), client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -77,7 +77,7 @@ def build_workflow(checkpoint_storage: FileCheckpointStorage): instructions=("You convert the research notes into a structured brief with milestones and risks."), client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) @@ -89,7 +89,7 @@ def build_workflow(checkpoint_storage: FileCheckpointStorage): instructions="You coordinate a team to complete complex tasks efficiently.", client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), ) diff --git a/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py b/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py index acfe43a750..e44e2a44ca 100644 --- a/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py +++ b/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py @@ -38,7 +38,7 @@ Plan review options: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -102,7 +102,7 @@ async def process_event_stream(stream: AsyncIterable[WorkflowEvent]) -> dict[str async def main() -> None: client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/sequential_agents.py b/python/samples/03-workflows/orchestrations/sequential_agents.py index 8a64b22368..70a25d9f58 100644 --- a/python/samples/03-workflows/orchestrations/sequential_agents.py +++ b/python/samples/03-workflows/orchestrations/sequential_agents.py @@ -30,7 +30,7 @@ Note on internal adapters: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -39,7 +39,7 @@ async def main() -> None: # 1) Create agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py b/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py index 2bef81ebe5..f4723a205d 100644 --- a/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py +++ b/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py @@ -3,8 +3,8 @@ import asyncio import os -from agent_framework import AgentResponseUpdate -from agent_framework.azure import AzureOpenAIResponsesClient +from agent_framework import Agent, AgentResponseUpdate +from agent_framework.foundry import FoundryChatClient from agent_framework.orchestrations import SequentialBuilder from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -25,8 +25,8 @@ Compare with `sequential_agents.py`, which uses the default behavior where the f conversation context is passed to each agent. Prerequisites: -- AZURE_AI_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for AzureOpenAIResponsesClient with required environment variables. +- FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. +- FOUNDRY_MODEL must be the deployment name of a model in your Foundry project. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -36,23 +36,26 @@ load_dotenv() async def main() -> None: # 1) Create agents - client = AzureOpenAIResponsesClient( - project_endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], - deployment_name=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) - writer = client.as_agent( + writer = Agent( + client=client, instructions="You are a concise copywriter. Provide a single, punchy marketing sentence based on the prompt.", name="writer", ) - translator = client.as_agent( + translator = Agent( + client=client, instructions="You are a translator. Translate the given text into French. Output only the translation.", name="translator", ) - reviewer = client.as_agent( + reviewer = Agent( + client=client, instructions="You are a reviewer. Evaluate the quality of the marketing tagline.", name="reviewer", ) diff --git a/python/samples/03-workflows/orchestrations/sequential_custom_executors.py b/python/samples/03-workflows/orchestrations/sequential_custom_executors.py index 79823ea643..a4fb2d602b 100644 --- a/python/samples/03-workflows/orchestrations/sequential_custom_executors.py +++ b/python/samples/03-workflows/orchestrations/sequential_custom_executors.py @@ -35,7 +35,7 @@ Custom executor contract: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. """ @@ -68,7 +68,7 @@ async def main() -> None: # 1) Create a content agent client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) content = Agent( diff --git a/python/samples/03-workflows/parallelism/fan_out_fan_in_edges.py b/python/samples/03-workflows/parallelism/fan_out_fan_in_edges.py index 3eaeb21ea0..456ff7e212 100644 --- a/python/samples/03-workflows/parallelism/fan_out_fan_in_edges.py +++ b/python/samples/03-workflows/parallelism/fan_out_fan_in_edges.py @@ -37,8 +37,8 @@ Show how to construct a parallel branch pattern in workflows. Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Familiarity with WorkflowBuilder, executors, edges, events, and streaming runs. -- Azure OpenAI access configured for FoundryChatClient. Log in with Azure CLI and set any required environment variables. - Comfort reading AgentExecutorResponse.agent_response.text for assistant output aggregation. """ @@ -118,7 +118,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -132,7 +132,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -146,7 +146,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/03-workflows/state-management/state_with_agents.py b/python/samples/03-workflows/state-management/state_with_agents.py index b9e800ba04..3c51b6fb9e 100644 --- a/python/samples/03-workflows/state-management/state_with_agents.py +++ b/python/samples/03-workflows/state-management/state_with_agents.py @@ -40,7 +40,7 @@ Show how to: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure OpenAI configured for FoundryChatClient with required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via azure-identity. Use AzureCliCredential and run az login before executing the sample. - Familiarity with WorkflowBuilder, executors, conditional edges, and streaming runs. """ @@ -165,7 +165,7 @@ def create_spam_detection_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -183,7 +183,7 @@ def create_email_assistant_agent() -> Agent: return Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/03-workflows/state-management/workflow_kwargs.py b/python/samples/03-workflows/state-management/workflow_kwargs.py index 630eaafc52..0d50b8710d 100644 --- a/python/samples/03-workflows/state-management/workflow_kwargs.py +++ b/python/samples/03-workflows/state-management/workflow_kwargs.py @@ -29,7 +29,7 @@ Key Concepts: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Environment variables configured +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. """ @@ -83,7 +83,7 @@ async def main() -> None: # Create chat client client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/tool-approval/concurrent_builder_tool_approval.py b/python/samples/03-workflows/tool-approval/concurrent_builder_tool_approval.py index d11e4d3525..b9a0e7d229 100644 --- a/python/samples/03-workflows/tool-approval/concurrent_builder_tool_approval.py +++ b/python/samples/03-workflows/tool-approval/concurrent_builder_tool_approval.py @@ -46,7 +46,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI or Azure OpenAI configured with the required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Basic familiarity with ConcurrentBuilder and streaming workflow events. """ @@ -136,7 +136,7 @@ async def main() -> None: # 3. Create two agents focused on different stocks but with the same tool sets client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/tool-approval/group_chat_builder_tool_approval.py b/python/samples/03-workflows/tool-approval/group_chat_builder_tool_approval.py index 8fff4b7dd3..371ff20294 100644 --- a/python/samples/03-workflows/tool-approval/group_chat_builder_tool_approval.py +++ b/python/samples/03-workflows/tool-approval/group_chat_builder_tool_approval.py @@ -45,7 +45,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI or Azure OpenAI configured with the required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Basic familiarity with GroupChatBuilder and streaming workflow events. """ @@ -136,7 +136,7 @@ async def main() -> None: # 3. Create specialized agents client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) diff --git a/python/samples/03-workflows/tool-approval/sequential_builder_tool_approval.py b/python/samples/03-workflows/tool-approval/sequential_builder_tool_approval.py index a6272b196c..318506316e 100644 --- a/python/samples/03-workflows/tool-approval/sequential_builder_tool_approval.py +++ b/python/samples/03-workflows/tool-approval/sequential_builder_tool_approval.py @@ -46,7 +46,7 @@ Demonstrate: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- OpenAI or Azure OpenAI configured with the required environment variables. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Basic familiarity with SequentialBuilder and streaming workflow events. """ @@ -109,7 +109,7 @@ async def main() -> None: # 2. Create the agent with tools (approval mode is set per-tool via decorator) client = FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ) database_agent = Agent( diff --git a/python/samples/03-workflows/visualization/concurrent_with_visualization.py b/python/samples/03-workflows/visualization/concurrent_with_visualization.py index f11b8d291b..d59268540e 100644 --- a/python/samples/03-workflows/visualization/concurrent_with_visualization.py +++ b/python/samples/03-workflows/visualization/concurrent_with_visualization.py @@ -34,7 +34,7 @@ What it does: Prerequisites: - FOUNDRY_PROJECT_ENDPOINT must be your Azure AI Foundry Agent Service (V2) project endpoint. -- Azure AI/ Azure OpenAI for `FoundryChatClient` agents. +- FOUNDRY_MODEL must be set to your Azure OpenAI model deployment name. - Authentication via `azure-identity` — uses `AzureCliCredential()` (run `az login`). - For visualization export: `pip install graphviz>=0.20.0` and install GraphViz binaries. """ @@ -100,7 +100,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -115,7 +115,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( @@ -130,7 +130,7 @@ async def main() -> None: Agent( client=FoundryChatClient( project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + model=os.environ["FOUNDRY_MODEL"], credential=AzureCliCredential(), ), instructions=( diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/.env.example b/python/samples/05-end-to-end/evaluation/foundry_evals/.env.example new file mode 100644 index 0000000000..b6a8af233e --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/.env.example @@ -0,0 +1,3 @@ +FOUNDRY_PROJECT_ENDPOINT="" +FOUNDRY_MODEL="" + diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/README.md b/python/samples/05-end-to-end/evaluation/foundry_evals/README.md new file mode 100644 index 0000000000..81412a7f0e --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/README.md @@ -0,0 +1,46 @@ +# Foundry Evals Integration Samples + +These samples demonstrate evaluating agent-framework agents using Azure AI Foundry's built-in evaluators. + +## Available Evaluators + +| Category | Evaluators | +|----------|-----------| +| **Agent behavior** | `intent_resolution`, `task_adherence`, `task_completion`, `task_navigation_efficiency` | +| **Tool usage** | `tool_call_accuracy`, `tool_selection`, `tool_input_accuracy`, `tool_output_utilization`, `tool_call_success` | +| **Quality** | `coherence`, `fluency`, `relevance`, `groundedness`, `response_completeness`, `similarity` | +| **Safety** | `violence`, `sexual`, `self_harm`, `hate_unfairness` | + +## Samples + +### `evaluate_agent_sample.py` — Dataset Evaluation (Path 3) + +The dev inner loop. Two patterns from simplest to most control: + +1. **`evaluate_agent()`** — One call: runs agent → converts → evaluates +2. **`FoundryEvals.evaluate()`** — Run agent yourself, convert with `AgentEvalConverter`, inspect/modify, then evaluate + +```bash +uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py +``` + +### `evaluate_traces_sample.py` — Trace & Response Evaluation (Path 1) + +Evaluate what already happened — zero changes to agent code: + +1. **`evaluate_traces(response_ids=...)`** — Evaluate Responses API responses by ID +2. **`evaluate_traces(agent_id=...)`** — Evaluate agent behavior from OTel traces in App Insights + +```bash +uv run samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py +``` + +## Setup + +Create a `.env` file with configuration as in the `.env.example` file in this folder. + +## Which sample should I start with? + +- **"I want to test my agent during development"** → `evaluate_agent_sample.py`, Pattern 1 +- **"I want to evaluate past agent runs"** → `evaluate_traces_sample.py` +- **"I want to inspect/modify eval data before submitting"** → `evaluate_agent_sample.py`, Pattern 2 diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py new file mode 100644 index 0000000000..1135077f5c --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_agent_sample.py @@ -0,0 +1,154 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate an agent using Azure AI Foundry's built-in evaluators. + +This sample demonstrates two patterns: +1. evaluate_agent(responses=...) — Evaluate a response you already have. +2. evaluate_agent(queries=...) — Run the agent against test queries and evaluate in one call. + +See ``evaluate_tool_calls_sample.py`` for tool-call accuracy evaluation. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework import Agent, ConversationSplit, evaluate_agent +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +# Define a simple tool for the agent +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + weather_data = { + "seattle": "62°F, cloudy with a chance of rain", + "london": "55°F, overcast", + "paris": "68°F, partly sunny", + } + return weather_data.get(location.lower(), f"Weather data not available for {location}") + + +def get_flight_price(origin: str, destination: str) -> str: + """Get the price of a flight between two cities.""" + return f"Flights from {origin} to {destination}: $450 round-trip" + + +async def main() -> None: + # 1. Set up the FoundryChatClient + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("FOUNDRY_MODEL", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # 2. Create an agent with tools + agent = Agent( + client=chat_client, + name="travel-assistant", + instructions=( + "You are a helpful travel assistant. Use your tools to answer questions about weather and flights." + ), + tools=[get_weather, get_flight_price], + ) + + # 3. Create the evaluator — provider config goes here, once + evals = FoundryEvals(client=chat_client) + + # ========================================================================= + # Pattern 1: evaluate_agent(responses=...) — evaluate a response you already have + # ========================================================================= + print("=" * 60) + print("Pattern 1: evaluate_agent(responses=...) — evaluate existing response") + print("=" * 60) + + query = "How much does a flight from Seattle to Paris cost?" + response = await agent.run(query) + print(f"Agent said: {response.text[:100]}...") + + # Pass agent= so tool definitions are extracted, queries= for the eval item context + results = await evaluate_agent( + agent=agent, + responses=response, + queries=[query], + evaluators=FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY], + ), + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + print(f"Portal: {r.report_url}") + if r.all_passed: + print("[PASS] All passed") + else: + print(f"[FAIL] {r.failed} failed") + + # ========================================================================= + # Pattern 2a: evaluate_agent() — batch test queries + # ========================================================================= + print() + print("=" * 60) + print("Pattern 2a: evaluate_agent()") + print("=" * 60) + + # Calls agent.run() under the covers for each query, then evaluates + results = await evaluate_agent( + agent=agent, + queries=[ + "What's the weather like in Seattle?", + "How much does a flight from Seattle to Paris cost?", + "What should I pack for London?", + ], + evaluators=evals, # uses smart defaults (auto-adds tool_call_accuracy) + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + print(f"Portal: {r.report_url}") + if r.all_passed: + print("[PASS] All passed") + else: + print(f"[FAIL] {r.failed} failed") + + # ========================================================================= + # Pattern 2b: evaluate_agent() — with conversation split override + # ========================================================================= + print() + print("=" * 60) + print("Pattern 2b: evaluate_agent() with conversation_split") + print("=" * 60) + + # conversation_split forces all evaluators to use the same split strategy. + # FULL evaluates the entire conversation trajectory against the original query. + results = await evaluate_agent( + agent=agent, + queries=[ + "What's the weather like in Seattle?", + "What should I pack for London?", + ], + evaluators=evals, + conversation_split=ConversationSplit.FULL, # overrides evaluator defaults + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + print(f"Portal: {r.report_url}") + if r.all_passed: + print("[PASS] All passed") + else: + print(f"[FAIL] {r.failed} failed") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_mixed_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_mixed_sample.py new file mode 100644 index 0000000000..b15781e2bd --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_mixed_sample.py @@ -0,0 +1,159 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Mix local and cloud evaluation providers in a single evaluate_agent() call. + +This sample demonstrates three patterns: +1. Local-only: Fast, API-free checks for inner-loop development. +2. Cloud-only: Full Foundry evaluators for comprehensive quality assessment. +3. Mixed: Local + Foundry evaluators in a single evaluate_agent() call. + +Mixing lets you get instant local feedback (keyword presence, tool usage) +alongside deeper cloud-based quality evaluation (relevance, coherence) +in one call. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and FOUNDRY_MODEL in .env +""" + +import asyncio +import os + +from agent_framework import ( + Agent, + LocalEvaluator, + evaluate_agent, + keyword_check, + tool_called_check, +) +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +# Define a simple tool for the agent +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + weather_data = { + "seattle": "62°F, cloudy with a chance of rain", + "london": "55°F, overcast", + "paris": "68°F, partly sunny", + } + return weather_data.get(location.lower(), f"Weather data not available for {location}") + + +async def main() -> None: + # 1. Set up the chat client + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # 2. Create an agent with a tool + agent = Agent( + client=chat_client, + name="weather-assistant", + instructions="You are a helpful weather assistant. Use the get_weather tool to answer questions.", + tools=[get_weather], + ) + + # ========================================================================= + # Pattern 1: Local evaluation only (no API calls, instant results) + # ========================================================================= + print("=" * 60) + print("Pattern 1: Local evaluation only") + print("=" * 60) + + local = LocalEvaluator( + keyword_check("weather", "seattle"), + tool_called_check("get_weather"), + ) + + results = await evaluate_agent( + agent=agent, + queries=["What's the weather in Seattle?"], + evaluators=local, + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + for check_name, counts in r.per_evaluator.items(): + print(f" {check_name}: {counts['passed']} passed, {counts['failed']} failed") + if r.all_passed: + print("[PASS] All local checks passed!") + else: + print(f"[FAIL] Failures: {r.error}") + + # ========================================================================= + # Pattern 2: Foundry evaluation only (cloud-based quality assessment) + # ========================================================================= + print() + print("=" * 60) + print("Pattern 2: Foundry evaluation only") + print("=" * 60) + + foundry = FoundryEvals(client=chat_client) + + results = await evaluate_agent( + agent=agent, + queries=["What's the weather in Seattle?"], + evaluators=foundry, + ) + + for r in results: + print(f"Status: {r.status}") + print(f"Results: {r.passed}/{r.total} passed") + print(f"Portal: {r.report_url}") + if r.all_passed: + print("[PASS] All passed") + else: + print(f"[FAIL] {r.failed} failed") + + # ========================================================================= + # Pattern 3: Mixed — local + Foundry in one call + # ========================================================================= + print() + print("=" * 60) + print("Pattern 3: Mixed local + Foundry evaluation") + print("=" * 60) + + # Local checks: fast smoke tests + local = LocalEvaluator( + keyword_check("weather"), + tool_called_check("get_weather"), + ) + + # Foundry: deep quality assessment + foundry = FoundryEvals(client=chat_client) + + # Pass both as a list — returns one EvalResults per provider + results = await evaluate_agent( + agent=agent, + queries=[ + "What's the weather in Seattle?", + "Tell me the weather in London", + ], + evaluators=[local, foundry], + ) + + for r in results: + status = "PASS" if r.all_passed else "FAIL" + print(f" {status} {r.provider}: {r.passed}/{r.total} passed") + for check_name, counts in r.per_evaluator.items(): + print(f" {check_name}: {counts['passed']}/{counts['passed'] + counts['failed']}") + if r.report_url: + print(f" Portal: {r.report_url}") + + if all(r.all_passed for r in results): + print("[PASS] All checks passed (local + Foundry)!") + else: + failed = [r.provider for r in results if not r.all_passed] + print(f"[FAIL] Failed providers: {', '.join(failed)}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_multiturn_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_multiturn_sample.py new file mode 100644 index 0000000000..b28bba22c0 --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_multiturn_sample.py @@ -0,0 +1,182 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate multi-turn conversations with different split strategies. + +The same multi-turn conversation can be split different ways, each evaluating +a different aspect of agent behavior: + +1. LAST_TURN (default) — "Was the last response good given context?" +2. FULL — "Did the whole conversation serve the original request?" +3. per_turn_items — "Was each individual response appropriate?" + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework import Content, ConversationSplit, EvalItem, FunctionTool, Message +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + +# A multi-turn conversation with tool calls that we'll evaluate three ways. +# Uses framework Message/Content types for type-safe conversation construction. +CONVERSATION: list[Message] = [ + # Turn 1: user asks about weather -> agent calls tool -> responds + Message("user", ["What's the weather in Seattle?"]), + Message( + "assistant", + [ + Content.from_function_call("c1", "get_weather", arguments={"location": "seattle"}), + ], + ), + Message( + "tool", + [ + Content.from_function_result("c1", result="62°F, cloudy with a chance of rain"), + ], + ), + Message("assistant", ["Seattle is 62°F, cloudy with a chance of rain."]), + # Turn 2: user asks about Paris -> agent calls tool -> responds + Message("user", ["And Paris?"]), + Message( + "assistant", + [ + Content.from_function_call("c2", "get_weather", arguments={"location": "paris"}), + ], + ), + Message( + "tool", + [ + Content.from_function_result("c2", result="68°F, partly sunny"), + ], + ), + Message("assistant", ["Paris is 68°F, partly sunny."]), + # Turn 3: user asks for comparison -> agent synthesizes without tool + Message("user", ["Can you compare them?"]), + Message( + "assistant", + [ + ( + "Seattle is cooler at 62°F with rain likely, while Paris is warmer " + "at 68°F and partly sunny. Paris is the better choice for outdoor activities." + ), + ], + ), +] + +TOOLS = [ + FunctionTool( + name="get_weather", + description="Get the current weather for a location.", + ), +] + + +def print_split(item: EvalItem, split: ConversationSplit = ConversationSplit.LAST_TURN) -> None: + """Print the query/response split for an EvalItem.""" + query_msgs, response_msgs = item.split_messages(split) + print(f" query_messages ({len(query_msgs)}):") + for m in query_msgs: + text = m.text or "" + print(f" {m.role}: {text[:70]}") + print(f" response_messages ({len(response_msgs)}):") + for m in response_msgs: + text = m.text or "" + print(f" {m.role}: {text[:70]}") + + +async def main() -> None: + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # ========================================================================= + # Strategy 1: LAST_TURN (default) + # "Given all context, was the last response good?" + # ========================================================================= + print("=" * 70) + print("Strategy 1: LAST_TURN — evaluate the final response") + print("=" * 70) + + # EvalItem takes conversation + tools; query/response are derived via split strategy + item = EvalItem(CONVERSATION, tools=TOOLS) + + print_split(item, ConversationSplit.LAST_TURN) + + results = await FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE], + # conversation_split defaults to LAST_TURN + ).evaluate([item], eval_name="Split Strategy: LAST_TURN") + + print(f"\n Result: {results.passed}/{results.total} passed") + print(f" Portal: {results.report_url}") + for ir in results.items: + for s in ir.scores: + print(f" {'PASS' if s.passed else 'FAIL'} {s.name}: {s.score}") + print() + + # ========================================================================= + # Strategy 2: FULL + # "Given the original request, did the whole conversation serve the user?" + # ========================================================================= + print("=" * 70) + print("Strategy 2: FULL — evaluate the entire conversation trajectory") + print("=" * 70) + + print_split(item, ConversationSplit.FULL) + + results = await FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE], + conversation_split=ConversationSplit.FULL, + ).evaluate([item], eval_name="Split Strategy: FULL") + + print(f"\n Result: {results.passed}/{results.total} passed") + print(f" Portal: {results.report_url}") + for ir in results.items: + for s in ir.scores: + print(f" {'PASS' if s.passed else 'FAIL'} {s.name}: {s.score}") + print() + + # ========================================================================= + # Strategy 3: per_turn_items + # "Was each individual response appropriate at that point?" + # ========================================================================= + print("=" * 70) + print("Strategy 3: per_turn_items — evaluate each turn independently") + print("=" * 70) + + items = EvalItem.per_turn_items(CONVERSATION, tools=TOOLS) + print(f" Split into {len(items)} items from {len(CONVERSATION)} messages:\n") + for i, it in enumerate(items): + print(f" Turn {i + 1}: query={it.query!r}, response={it.response[:60]!r}...") + print() + + results = await FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE], + ).evaluate(items, eval_name="Split Strategy: Per-Turn") + + print(f"\n Result: {results.passed}/{results.total} passed ({len(items)} items × 2 evaluators)") + print(f" Portal: {results.report_url}") + for ir in results.items: + for s in ir.scores: + print(f" {'PASS' if s.passed else 'FAIL'} {s.name}: {s.score}") + print() + + print("=" * 70) + print("All strategies complete. Compare results in the Foundry portal.") + print("=" * 70) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py new file mode 100644 index 0000000000..4b5d892fe4 --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_tool_calls_sample.py @@ -0,0 +1,89 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate tool-calling accuracy using Azure AI Foundry's TOOL_CALL_ACCURACY evaluator. + +This sample demonstrates evaluating how well an agent selects and invokes tools +by using ``FoundryEvals.evaluate()`` with ``TOOL_CALL_ACCURACY``. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework import Agent, AgentEvalConverter +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + weather_data = { + "seattle": "62°F, cloudy with a chance of rain", + "london": "55°F, overcast", + "paris": "68°F, partly sunny", + } + return weather_data.get(location.lower(), f"Weather data not available for {location}") + + +def get_flight_price(origin: str, destination: str) -> str: + """Get the price of a flight between two cities.""" + return f"Flights from {origin} to {destination}: $450 round-trip" + + +async def main() -> None: + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # Create an agent with tools + agent = Agent( + client=chat_client, + name="travel-assistant", + instructions=( + "You are a helpful travel assistant. " + "Use your tools to answer questions about weather and flights." + ), + tools=[get_weather, get_flight_price], + ) + + # Run the agent and convert responses to eval items + queries = [ + "What's the weather in Paris?", + "Find me a flight from London to Seattle", + ] + + items = [] + for q in queries: + response = await agent.run(q) + print(f"Query: {q}") + print(f"Response: {response.text[:100]}...") + + item = AgentEvalConverter.to_eval_item(query=q, response=response, agent=agent) + items.append(item) + + print(f" Has tools: {item.tools is not None}") + if item.tools: + print(f" Tools: {[t.name for t in item.tools]}") + + # Submit to Foundry with tool_call_accuracy evaluator + evals = FoundryEvals( + client=chat_client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TOOL_CALL_ACCURACY], + ) + results = await evals.evaluate(items, eval_name="Tool Call Accuracy Eval") + + print(f"\nStatus: {results.status}") + print(f"Results: {results.passed}/{results.total} passed") + print(f"Portal: {results.report_url}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py new file mode 100644 index 0000000000..e0d4d07950 --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_traces_sample.py @@ -0,0 +1,114 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate agent responses that already exist in Foundry (zero-code-change). + +This sample demonstrates two patterns: +1. evaluate_traces(response_ids=...) — Evaluate specific Responses API responses by ID. +2. evaluate_traces(agent_id=...) — Evaluate agent behavior from OTel traces in App Insights. + +These are the "zero-code-change" evaluation paths — the agent has already run, +and you're evaluating what happened after the fact. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Response IDs from prior agent runs (for Pattern 1) +- OTel traces exported to App Insights (for Pattern 2) +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework.foundry import FoundryChatClient, FoundryEvals, evaluate_traces +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +async def main() -> None: + # 1. Set up the chat client + chat_client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # ========================================================================= + # Pattern 1: evaluate_traces(response_ids=...) — By response ID + # ========================================================================= + # If your agent uses the Responses API (e.g., FoundryChatClient), + # each run produces a response_id. Pass those IDs to evaluate_traces() + # and Foundry retrieves the full conversation for evaluation. + print("=" * 60) + print("Pattern 1: evaluate_traces(response_ids=...)") + print("=" * 60) + + # Replace these with actual response IDs from your agent runs + response_ids = [ + "resp_abc123", + "resp_def456", + ] + + results = await evaluate_traces( + response_ids=response_ids, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.GROUNDEDNESS, FoundryEvals.TOOL_CALL_ACCURACY], + client=chat_client, + ) + + print(f"Status: {results.status}") + print(f"Results: {results.result_counts}") + print(f"Portal: {results.report_url}") + + # ========================================================================= + # Pattern 2: evaluate_traces(response_ids=...) — Batch response evaluation + # ========================================================================= + # Evaluate multiple prior responses by their IDs. This uses the same + # response-based data source under the covers but lets you batch them. + # + # A future trace-based pattern (agent_id + lookback_hours) is shown + # commented out below — it requires OTel traces exported to App Insights. + print() + print("=" * 60) + print("Pattern 2: evaluate_traces(response_ids=...)") + print("=" * 60) + + # Evaluate by response IDs (uses response-based data source internally) + results = await evaluate_traces( + response_ids=response_ids, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.COHERENCE], + client=chat_client, + ) + + print(f"Status: {results.status}") + print(f"Portal: {results.report_url}") + + # Evaluate by agent ID + time window (when trace-based API is available) + # results = await evaluate_traces( + # agent_id="travel-bot", + # evaluators=[FoundryEvals.INTENT_RESOLUTION, FoundryEvals.TASK_ADHERENCE], + # client=chat_client, + # lookback_hours=24, + # ) + + +if __name__ == "__main__": + asyncio.run(main()) + + +""" +Sample output (with actual Azure AI Foundry project and valid response IDs): + +============================================================ +Pattern 1: evaluate_traces(response_ids=...) +============================================================ +Status: completed +Results: {'passed': 2, 'failed': 0, 'errored': 0} +Portal: https://ai.azure.com/... + +============================================================ +Pattern 2: evaluate_traces(response_ids=...) +============================================================ +Status: completed +Portal: https://ai.azure.com/... +""" diff --git a/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py new file mode 100644 index 0000000000..5f0a7315dc --- /dev/null +++ b/python/samples/05-end-to-end/evaluation/foundry_evals/evaluate_workflow_sample.py @@ -0,0 +1,176 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Evaluate a multi-agent workflow using Azure AI Foundry evaluators. + +This sample demonstrates two patterns: +1. Post-hoc: Run the workflow, then evaluate the result you already have. +2. Run + evaluate: Pass queries and let evaluate_workflow() run the workflow for you. + +Both patterns return a list of results (one per provider), each with a per-agent +breakdown in sub_results so you can identify which agent is underperforming. + +Prerequisites: +- An Azure AI Foundry project with a deployed model +- Set FOUNDRY_PROJECT_ENDPOINT and AZURE_AI_MODEL_DEPLOYMENT_NAME in .env +""" + +import asyncio +import os + +from agent_framework import Agent, evaluate_workflow +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from agent_framework_orchestrations import SequentialBuilder +from azure.identity import AzureCliCredential +from dotenv import load_dotenv + +load_dotenv() + + +# Simple tools for the agents +def get_weather(location: str) -> str: + """Get the current weather for a location.""" + weather_data = { + "seattle": "62°F, cloudy with a chance of rain", + "london": "55°F, overcast", + "paris": "68°F, partly sunny", + } + return weather_data.get(location.lower(), f"Weather data not available for {location}") + + +def get_flight_price(origin: str, destination: str) -> str: + """Get the price of a flight between two cities.""" + return f"Flights from {origin} to {destination}: $450 round-trip" + + +async def main() -> None: + # 1. Set up the chat client + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o"), + credential=AzureCliCredential(), + ) + + # 2. Create agents for a sequential workflow + # Use store=False so agents don't chain conversation state via previous_response_id. + # This allows the workflow to be run multiple times without stale state issues. + researcher = Agent( + client=client, + name="researcher", + instructions=( + "You are a travel researcher. Use your tools to gather weather " + "and flight information for the destination the user asks about." + ), + tools=[get_weather, get_flight_price], + default_options={"store": False}, + ) + + planner = Agent( + client=client, + name="planner", + instructions=( + "You are a travel planner. Based on the research provided, " + "create a concise travel recommendation with packing tips." + ), + default_options={"store": False}, + ) + + # 3. Build a sequential workflow: researcher -> planner + workflow = SequentialBuilder(participants=[researcher, planner]).build() + + # 4. Create the evaluator — provider config goes here, once + evals = FoundryEvals(client=client) + + # ========================================================================= + # Pattern 1: Post-hoc — evaluate a workflow run you already did + # ========================================================================= + print("=" * 60) + print("Pattern 1: Post-hoc workflow evaluation") + print("=" * 60) + + result = await workflow.run("Plan a trip from Seattle to Paris") + + eval_results = await evaluate_workflow( + workflow=workflow, + workflow_result=result, + evaluators=evals, + ) + + for r in eval_results: + print(f"\nOverall: {r.status}") + print(f" Passed: {r.passed}/{r.total}") + print(f" Portal: {r.report_url}") + + print("\nPer-agent breakdown:") + for agent_name, agent_eval in r.sub_results.items(): + print(f" {agent_name}: {agent_eval.passed}/{agent_eval.total} passed") + if agent_eval.report_url: + print(f" Portal: {agent_eval.report_url}") + + # ========================================================================= + # Pattern 2: Run + evaluate with multiple queries + # ========================================================================= + # Build a fresh workflow to avoid stale session state from Pattern 1. + # The Responses API tracks previous_response_id per session, so reusing + # a workflow after a run would reference stale tool calls. + workflow2 = SequentialBuilder(participants=[researcher, planner]).build() + + print() + print("=" * 60) + print("Pattern 2: Run + evaluate with multiple queries") + print("=" * 60) + + eval_results = await evaluate_workflow( + workflow=workflow2, + queries=[ + "Plan a trip from London to Tokyo", + "Plan a trip from New York to Rome", + ], + evaluators=FoundryEvals( + client=client, + evaluators=[FoundryEvals.RELEVANCE, FoundryEvals.TASK_ADHERENCE], + ), + ) + + for r in eval_results: + print(f"\nOverall: {r.status}") + print(f" Passed: {r.passed}/{r.total}") + if r.report_url: + print(f" Portal: {r.report_url}") + + print("\nPer-agent breakdown:") + for agent_name, agent_eval in r.sub_results.items(): + print(f" {agent_name}: {agent_eval.passed}/{agent_eval.total} passed") + if agent_eval.report_url: + print(f" Portal: {agent_eval.report_url}") + + +if __name__ == "__main__": + asyncio.run(main()) + + +""" +Sample output (with actual Azure AI Foundry project): + +============================================================ +Pattern 1: Post-hoc workflow evaluation +============================================================ + +Overall: completed + Passed: 2/2 + Portal: https://ai.azure.com/... + +Per-agent breakdown: + researcher: 1/1 passed + planner: 1/1 passed + +============================================================ +Pattern 2: Run + evaluate with multiple queries +============================================================ + +Overall: completed + Passed: 4/4 + +Per-agent breakdown: + researcher: 2/2 passed + planner: 2/2 passed +""" diff --git a/python/samples/05-end-to-end/evaluation/red_teaming/README.md b/python/samples/05-end-to-end/evaluation/red_teaming/README.md index 39fda91ae4..9a9efaafe6 100644 --- a/python/samples/05-end-to-end/evaluation/red_teaming/README.md +++ b/python/samples/05-end-to-end/evaluation/red_teaming/README.md @@ -174,7 +174,7 @@ Overall Attack Success Rate: 7.2% - [Azure AI Evaluation SDK](https://learn.microsoft.com/azure/ai-foundry/how-to/develop/evaluate-sdk) - [Risk and Safety Evaluations](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-metrics-built-in#risk-and-safety-evaluators) - [Azure AI Red Teaming Notebook](https://github.com/Azure-Samples/azureai-samples/blob/main/scenarios/evaluate/AI_RedTeaming/AI_RedTeaming.ipynb) -- [PyRIT - Python Risk Identification Toolkit](https://github.com/Azure/PyRIT) +- [PyRIT - Python Risk Identification Toolkit](https://github.com/microsoft/PyRIT) ## Troubleshooting diff --git a/python/samples/05-end-to-end/evaluation/self_reflection/.env.example b/python/samples/05-end-to-end/evaluation/self_reflection/.env.example index 413a62c0ff..8c24539c3c 100644 --- a/python/samples/05-end-to-end/evaluation/self_reflection/.env.example +++ b/python/samples/05-end-to-end/evaluation/self_reflection/.env.example @@ -1,3 +1 @@ -AZURE_OPENAI_ENDPOINT="..." -AZURE_OPENAI_API_KEY="..." -AZURE_AI_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects//" +FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com diff --git a/python/samples/05-end-to-end/evaluation/self_reflection/README.md b/python/samples/05-end-to-end/evaluation/self_reflection/README.md index 5c26f352e7..0591f37f73 100644 --- a/python/samples/05-end-to-end/evaluation/self_reflection/README.md +++ b/python/samples/05-end-to-end/evaluation/self_reflection/README.md @@ -6,31 +6,27 @@ This sample demonstrates the self-reflection pattern using Agent Framework and A **What it demonstrates:** - Iterative self-reflection loop that automatically improves responses based on groundedness evaluation +- Using `FoundryEvals` to score each iteration via the Foundry Groundedness evaluator - Batch processing of prompts from JSONL files with progress tracking -- Using `AzureOpenAIResponsesClient` with a Project Endpoint and Azure CLI authentication +- Using `FoundryChatClient` with a Project Endpoint and Azure CLI authentication - Comprehensive summary statistics and detailed result tracking ## Prerequisites ### Azure Resources -- **Azure OpenAI Responses in Foundry**: Deploy models (default: gpt-5.2 for both agent and judge) +- **Azure AI Foundry project**: Deploy models (default: gpt-5.2 for both agent and judge) - **Azure CLI**: Run `az login` to authenticate -### Python Environment -```bash -pip install agent-framework-core pandas --pre -``` - ### Environment Variables ```bash -AZURE_AI_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects// +FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com ``` ## Running the Sample ```bash # Basic usage -python self_reflection.py +uv run python samples/05-end-to-end/evaluation/self_reflection/self_reflection.py # With options python self_reflection.py --input my_prompts.jsonl \ @@ -42,8 +38,8 @@ python self_reflection.py --input my_prompts.jsonl \ **CLI Options:** - `--input`, `-i`: Input JSONL file - `--output`, `-o`: Output JSONL file -- `--agent-model`, `-m`: Agent model name (default: gpt-4.1) -- `--judge-model`, `-e`: Evaluator model name (default: gpt-4.1) +- `--agent-model`, `-m`: Agent model name (default: gpt-5.2) +- `--judge-model`, `-e`: Evaluator model name (default: gpt-5.2) - `--max-reflections`: Max iterations (default: 3) - `--limit`, `-n`: Process only first N prompts @@ -51,7 +47,7 @@ python self_reflection.py --input my_prompts.jsonl \ The agent iteratively improves responses: 1. Generate initial response -2. Evaluate groundedness (1-5 scale) +2. Evaluate groundedness via `FoundryEvals` (1-5 scale) 3. If score < 5, provide feedback and retry 4. Stop at max iterations or perfect score (5/5) @@ -70,7 +66,7 @@ In the Foundry UI, under `Build`/`Evaluations` you can view detailed results for - Context - Query - Response -- Groundedness scores and reasoning for each interation of each prompt +- Groundedness scores and reasoning for each iteration of each prompt ## Related Resources diff --git a/python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py b/python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py index 8251e89e72..a96841b02a 100644 --- a/python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py +++ b/python/samples/05-end-to-end/evaluation/self_reflection/self_reflection.py @@ -17,26 +17,20 @@ import time from pathlib import Path from typing import Any -import openai import pandas as pd -from agent_framework import Agent, Message -from agent_framework.foundry import FoundryChatClient -from azure.ai.projects import AIProjectClient -from azure.identity import AzureCliCredential +from agent_framework import Agent, EvalItem, Message +from agent_framework.foundry import FoundryChatClient, FoundryEvals +from azure.identity.aio import AzureCliCredential as AsyncAzureCliCredential from dotenv import load_dotenv -from openai.types.eval_create_params import DataSourceConfigCustom -from openai.types.evals.create_eval_jsonl_run_data_source_param import ( - CreateEvalJSONLRunDataSourceParam, - SourceFileContent, - SourceFileContentContent, -) """ Self-Reflection LLM Runner Reflexion: language agents with verbal reinforcement learning. Noah Shinn, Federico Cassano, Ashwin Gopinath, Karthik Narasimhan, and Shunyu Yao. 2023. -In Proceedings of the 37th International Conference on Neural Information Processing Systems (NIPS '23). Curran Associates Inc., Red Hook, NY, USA, Article 377, 8634–8652. +In Proceedings of the 37th International Conference on Neural Information +Processing Systems (NIPS '23). Curran Associates Inc., Red Hook, NY, USA, +Article 377, 8634–8652. https://arxiv.org/abs/2303.11366 This module implements a self-reflection loop for LLM responses using groundedness evaluation. @@ -59,8 +53,8 @@ Usage as CLI with extra options: SUMMARY ============================================================ Total prompts processed: 31 - ✓ Successful: 30 - ✗ Failed: 1 + [PASS] Successful: 30 + [FAIL] Failed: 1 Groundedness Scores: Average best score: 4.77/5 @@ -77,7 +71,7 @@ Iteration Statistics: Best on first try: 25/30 (83.3%) ============================================================ -✓ Processing complete! +[PASS] Processing complete! """ @@ -86,104 +80,37 @@ DEFAULT_AGENT_MODEL = "gpt-5.2" DEFAULT_JUDGE_MODEL = "gpt-5.2" -def create_openai_client(): - endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - credential = AzureCliCredential() - project_client = AIProjectClient(endpoint=endpoint, credential=credential) - return project_client.get_openai_client() - - -def create_async_project_client(): - from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient - from azure.identity.aio import AzureCliCredential as AsyncAzureCliCredential - - return AsyncAIProjectClient(endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], credential=AsyncAzureCliCredential()) - - -def create_eval(client: openai.OpenAI, judge_model: str) -> openai.types.EvalCreateResponse: - print("Creating Eval") - data_source_config = DataSourceConfigCustom({ - "type": "custom", - "item_schema": { - "type": "object", - "properties": { - "query": {"type": "string"}, - "response": {"type": "string"}, - "context": {"type": "string"}, - }, - "required": [], - }, - "include_sample_schema": True, - }) - - testing_criteria = [ - { - "type": "azure_ai_evaluator", - "name": "groundedness", - "evaluator_name": "builtin.groundedness", - "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}", "context": "{{item.context}}"}, - "initialization_parameters": {"deployment_name": f"{judge_model}"}, - } - ] - - return client.evals.create( - name="Eval", - data_source_config=data_source_config, - testing_criteria=testing_criteria, # type: ignore - ) - - -def run_eval( - client: openai.OpenAI, - eval_object: openai.types.EvalCreateResponse, +async def evaluate_groundedness( + evals: FoundryEvals, query: str, response: str, context: str, -): - eval_run_object = client.evals.runs.create( - eval_id=eval_object.id, - name="inline_data_run", - metadata={"team": "eval-exp", "scenario": "inline-data-v1"}, - data_source=CreateEvalJSONLRunDataSourceParam( - type="jsonl", - source=SourceFileContent( - type="file_content", - content=[ - SourceFileContentContent( - item={ - "query": query, - "context": context, - "response": response, - } - ), - ], - ), - ), +) -> float | None: + """Run a single groundedness evaluation and return the score.""" + item = EvalItem( + conversation=[ + Message("user", [query]), + Message("assistant", [response]), + ], + context=context, ) - - eval_run_response = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) - - MAX_RETRY = 10 - for _ in range(0, MAX_RETRY): - run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id) - if run.status == "failed": - print( - f"Eval run failed. Run ID: {run.id}, Status: {run.status}, Error: {getattr(run, 'error', 'Unknown error')}" - ) - continue - if run.status == "completed": - return list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) - time.sleep(5) - - print("Eval result retrieval timeout.") + results = await evals.evaluate( + [item], + eval_name="Self-Reflection Groundedness", + ) + if results.status != "completed" or not results.items: + return None + # Return the first evaluator score + for score in results.items[0].scores: + if score.score is not None: + return float(score.score) return None async def execute_query_with_self_reflection( *, - client: openai.OpenAI, + evals: FoundryEvals, agent: Agent, - eval_object: openai.types.EvalCreateResponse, full_user_query: str, context: str, max_self_reflections: int = 3, @@ -192,10 +119,10 @@ async def execute_query_with_self_reflection( Execute a query with self-reflection loop. Args: + evals: FoundryEvals instance for groundedness scoring agent: Agent instance to use for generating responses full_user_query: Complete prompt including system prompt, user request, and context context: Context document for groundedness evaluation - evaluator: Groundedness evaluator function max_self_reflections: Maximum number of self-reflection iterations Returns: @@ -205,7 +132,6 @@ async def execute_query_with_self_reflection( - best_iteration: Iteration number where best score was achieved - iteration_scores: List of groundedness scores for each iteration - messages: Full conversation history - - usage_metadata: Token usage information - num_retries: Number of iterations performed - total_groundedness_eval_time: Time spent on evaluations (seconds) - total_end_to_end_time: Total execution time (seconds) @@ -219,7 +145,7 @@ async def execute_query_with_self_reflection( raw_response = None total_groundedness_eval_time = 0.0 start_time = time.time() - iteration_scores = [] # Store all iteration scores in structured format + iteration_scores = [] for i in range(max_self_reflections): print(f" Self-reflection iteration {i + 1}/{max_self_reflections}...") @@ -227,22 +153,16 @@ async def execute_query_with_self_reflection( raw_response = await agent.run(messages=messages) agent_response = raw_response.text - # Evaluate groundedness + # Evaluate groundedness using FoundryEvals start_time_eval = time.time() - eval_run_output_items = run_eval( - client=client, - eval_object=eval_object, - query=full_user_query, - response=agent_response, - context=context, - ) - if eval_run_output_items is None: - print(f" ⚠️ Groundedness evaluation failed (timeout or error) for iteration {i + 1}.") - continue - score = eval_run_output_items[0].results[0].score + score = await evaluate_groundedness(evals, full_user_query, agent_response, context) end_time_eval = time.time() total_groundedness_eval_time += end_time_eval - start_time_eval + if score is None: + print(f" ⚠️ Groundedness evaluation failed for iteration {i + 1}.") + continue + # Store score in structured format iteration_scores.append(score) @@ -252,15 +172,15 @@ async def execute_query_with_self_reflection( # Update best response if improved if score > best_score: if best_score > 0: - print(f" ✓ Score improved from {best_score} to {score}/{max_score}") + print(f" [PASS] Score improved from {best_score} to {score}/{max_score}") best_score = score best_response = agent_response best_iteration = i + 1 if score == max_score: - print(" ✓ Perfect groundedness score achieved!") + print(" [PASS] Perfect groundedness score achieved!") break else: - print(f" → No improvement (score: {score}/{max_score}). Trying again...") + print(f" -> No improvement (score: {score}/{max_score}). Trying again...") # Add to conversation history messages.append(Message("assistant", [agent_response])) @@ -293,7 +213,6 @@ async def execute_query_with_self_reflection( async def run_self_reflection_batch( - project_client: AIProjectClient, input_file: str, output_file: str, agent_model: str = DEFAULT_AGENT_MODEL, @@ -301,7 +220,7 @@ async def run_self_reflection_batch( max_self_reflections: int = 3, env_file: str | None = None, limit: int | None = None, -): +) -> None: """ Run self-reflection on a batch of prompts. @@ -315,17 +234,36 @@ async def run_self_reflection_batch( limit: Optional limit to process only the first N prompts """ # Load environment variables - if env_file and os.path.exists(env_file): + if env_file: + if not os.path.isfile(env_file): + raise FileNotFoundError(f"Env file not found: {env_file}") load_dotenv(env_file, override=True) else: load_dotenv(override=True) - # Create agent, it loads environment variables AZURE_OPENAI_API_KEY and AZURE_OPENAI_ENDPOINT automatically - responses_client = FoundryChatClient( + from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient + + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + credential = AsyncAzureCliCredential() + project_client = AsyncAIProjectClient(endpoint=endpoint, credential=credential) + + # Create agent client + agent_client = FoundryChatClient( project_client=project_client, model=agent_model, ) + # Create FoundryEvals for groundedness scoring + judge_client = FoundryChatClient( + project_client=project_client, + model=judge_model, + ) + evals = FoundryEvals( + client=judge_client, + model=judge_model, + evaluators=[FoundryEvals.GROUNDEDNESS], + ) + # Load input data input_path = (Path(__file__).parent / input_file).resolve() print(f"Loading prompts from: {input_path}") @@ -351,13 +289,6 @@ async def run_self_reflection_batch( if missing_columns: raise ValueError(f"Input file missing required columns: {missing_columns}") - # Configure clients - print("Configuring Azure OpenAI client...") - client = create_openai_client() - - # Create Eval - eval_object = create_eval(client=client, judge_model=judge_model) - # Process each prompt print(f"Max self-reflections: {max_self_reflections}\n") @@ -367,9 +298,8 @@ async def run_self_reflection_batch( try: result = await execute_query_with_self_reflection( - client=client, - agent=Agent(client=responses_client, instructions=row["system_instruction"]), - eval_object=eval_object, + evals=evals, + agent=Agent(client=agent_client, instructions=row["system_instruction"]), full_user_query=row["full_prompt"], context=row["context_document"], max_self_reflections=max_self_reflections, @@ -393,13 +323,13 @@ async def run_self_reflection_batch( results.append(result_data) print( - f" ✓ Completed with score: {result['best_response_score']}/5 " + f" [PASS] Completed with score: {result['best_response_score']}/5 " f"(best at iteration {result['best_iteration']}/{result['num_retries']}, " f"time: {result['total_end_to_end_time']:.1f}s)\n" ) except Exception as e: - print(f" ✗ Error: {str(e)}\n") + print(f" [FAIL] Error: {str(e)}\n") # Save error information error_data = { @@ -434,8 +364,8 @@ async def run_self_reflection_batch( print("SUMMARY") print("=" * 60) print(f"Total prompts processed: {len(results_df)}") - print(f" ✓ Successful: {len(successful_runs)}") - print(f" ✗ Failed: {len(failed_runs)}") + print(f" [PASS] Successful: {len(successful_runs)}") + print(f" [FAIL] Failed: {len(failed_runs)}") if len(successful_runs) > 0: # Extract scores and iteration data from nested agent_response dict @@ -452,9 +382,8 @@ async def run_self_reflection_batch( perfect_scores = sum(1 for s in best_scores if s == 5) print("\nGroundedness Scores:") print(f" Average best score: {avg_score:.2f}/5") - print( - f" Perfect scores (5/5): {perfect_scores}/{len(best_scores)} ({100 * perfect_scores / len(best_scores):.1f}%)" - ) + pct = 100 * perfect_scores / len(best_scores) + print(f" Perfect scores (5/5): {perfect_scores}/{len(best_scores)} ({pct:.1f}%)") # Calculate improvement metrics if iteration_scores_list: @@ -472,9 +401,8 @@ async def run_self_reflection_batch( print(f" Average first score: {avg_first_score:.2f}/5") print(f" Average final score: {avg_last_score:.2f}/5") print(f" Average improvement: +{avg_improvement:.2f}") - print( - f" Responses that improved: {improved_count}/{len(improvements)} ({100 * improved_count / len(improvements):.1f}%)" - ) + pct = 100 * improved_count / len(improvements) + print(f" Responses that improved: {improved_count}/{len(improvements)} ({pct:.1f}%)") # Show iteration statistics if iterations: @@ -486,6 +414,8 @@ async def run_self_reflection_batch( print("=" * 60) + await credential.close() + async def main(): """CLI entry point.""" @@ -519,7 +449,6 @@ async def main(): # Run the batch processing try: await run_self_reflection_batch( - project_client=create_async_project_client(), input_file=args.input, output_file=args.output, agent_model=args.agent_model, @@ -528,10 +457,10 @@ async def main(): env_file=args.env_file, limit=args.limit, ) - print("\n✓ Processing complete!") + print("\n[PASS] Processing complete!") except Exception as e: - print(f"\n✗ Error: {str(e)}") + print(f"\n[FAIL] Error: {str(e)}") return 1 return 0 diff --git a/python/samples/README.md b/python/samples/README.md index 82a008504c..33a385a4f9 100644 --- a/python/samples/README.md +++ b/python/samples/README.md @@ -44,8 +44,8 @@ Samples call `load_dotenv()` to automatically load environment variables from a **Option 2: Export environment variables directly**: ```bash -export AZURE_AI_PROJECT_ENDPOINT="your-foundry-project-endpoint" -export AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME="gpt-4o" +export FOUNDRY_PROJECT_ENDPOINT="your-foundry-project-endpoint" +export FOUNDRY_MODEL="gpt-4o" ``` **Option 3: Using `env_file_path` parameter** (for per-client configuration): @@ -73,8 +73,8 @@ you pass an explicit Azure input. For the getting-started samples, you'll need at minimum: ```bash -AZURE_AI_PROJECT_ENDPOINT="your-foundry-project-endpoint" -AZURE_OPENAI_RESPONSES_DEPLOYMENT_NAME="gpt-4o" +FOUNDRY_PROJECT_ENDPOINT="your-foundry-project-endpoint" +FOUNDRY_MODEL="gpt-4o" ``` **Note for production**: In production environments, set environment variables through your deployment platform (e.g., Azure App Settings, Kubernetes ConfigMaps/Secrets) rather than using `.env` files. The `load_dotenv()` call in samples will have no effect when a `.env` file is not present, allowing environment variables to be loaded from the system. diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 4cffd5c71b..44a7e5a0c5 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -292,7 +292,7 @@ class CreateConcurrentValidationWorkflowExecutor(Executor): instructions=AgentInstruction, default_options={ "on_permission_request": prompt_permission, - "timeout": 60, + "timeout": 120, }, # type: ignore ) agents.append(agent) diff --git a/python/uv.lock b/python/uv.lock index 317113161f..590a78c791 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -94,7 +94,7 @@ wheels = [ [[package]] name = "agent-framework" -version = "1.0.0rc5" +version = "1.0.0rc6" source = { virtual = "." } dependencies = [ { name = "agent-framework-core", extra = ["all"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -147,7 +147,7 @@ dev = [ [[package]] name = "agent-framework-a2a" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/a2a" } dependencies = [ { name = "a2a-sdk", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -162,7 +162,7 @@ requires-dist = [ [[package]] name = "agent-framework-ag-ui" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/ag-ui" } dependencies = [ { name = "ag-ui-protocol", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -190,7 +190,7 @@ provides-extras = ["dev"] [[package]] name = "agent-framework-anthropic" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/anthropic" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -205,7 +205,7 @@ requires-dist = [ [[package]] name = "agent-framework-azure-ai" -version = "1.0.0rc5" +version = "1.0.0rc6" source = { editable = "packages/azure-ai" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -230,7 +230,7 @@ requires-dist = [ [[package]] name = "agent-framework-azure-ai-search" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/azure-ai-search" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -245,7 +245,7 @@ requires-dist = [ [[package]] name = "agent-framework-azure-cosmos" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/azure-cosmos" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -260,7 +260,7 @@ requires-dist = [ [[package]] name = "agent-framework-azurefunctions" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/azurefunctions" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -282,7 +282,7 @@ dev = [] [[package]] name = "agent-framework-bedrock" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/bedrock" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -299,7 +299,7 @@ requires-dist = [ [[package]] name = "agent-framework-chatkit" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/chatkit" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -314,7 +314,7 @@ requires-dist = [ [[package]] name = "agent-framework-claude" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/claude" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -329,7 +329,7 @@ requires-dist = [ [[package]] name = "agent-framework-copilotstudio" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/copilotstudio" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -344,7 +344,7 @@ requires-dist = [ [[package]] name = "agent-framework-core" -version = "1.0.0rc5" +version = "1.0.0rc6" source = { editable = "packages/core" } dependencies = [ { name = "opentelemetry-api", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -416,7 +416,7 @@ provides-extras = ["all"] [[package]] name = "agent-framework-declarative" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/declarative" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -441,7 +441,7 @@ dev = [{ name = "types-pyyaml", specifier = "==6.0.12.20250915" }] [[package]] name = "agent-framework-devui" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/devui" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -479,7 +479,7 @@ provides-extras = ["dev", "all"] [[package]] name = "agent-framework-durabletask" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/durabletask" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -506,7 +506,7 @@ dev = [{ name = "types-python-dateutil", specifier = "==2.9.0.20260305" }] [[package]] name = "agent-framework-foundry" -version = "1.0.0rc5" +version = "1.0.0rc6" source = { editable = "packages/foundry" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -523,7 +523,7 @@ requires-dist = [ [[package]] name = "agent-framework-foundry-local" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/foundry_local" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -540,7 +540,7 @@ requires-dist = [ [[package]] name = "agent-framework-github-copilot" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/github_copilot" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -555,7 +555,7 @@ requires-dist = [ [[package]] name = "agent-framework-lab" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/lab" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -636,7 +636,7 @@ dev = [ [[package]] name = "agent-framework-mem0" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/mem0" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -651,7 +651,7 @@ requires-dist = [ [[package]] name = "agent-framework-ollama" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/ollama" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -666,7 +666,7 @@ requires-dist = [ [[package]] name = "agent-framework-openai" -version = "1.0.0rc5" +version = "1.0.0rc6" source = { editable = "packages/openai" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -683,7 +683,7 @@ requires-dist = [ [[package]] name = "agent-framework-orchestrations" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/orchestrations" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -694,7 +694,7 @@ requires-dist = [{ name = "agent-framework-core", editable = "packages/core" }] [[package]] name = "agent-framework-purview" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/purview" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -711,7 +711,7 @@ requires-dist = [ [[package]] name = "agent-framework-redis" -version = "1.0.0b260319" +version = "1.0.0b260330" source = { editable = "packages/redis" } dependencies = [ { name = "agent-framework-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" },