From 3bbc81554b864e73a32680db5ff4623eb8228dba Mon Sep 17 00:00:00 2001 From: Evan Mattson <35585003+moonbox3@users.noreply.github.com> Date: Tue, 19 May 2026 09:15:25 +0900 Subject: [PATCH] Python: Improve the handling of intermediate outputs for workflows and orchestrations (#5623) * Improve the handling of intermediate outputs for workflows and orchestrations * Address PR review feedback on intermediate output forwarding - Switch workflow.as_agent() forwarding to an explicit allowlist of {output, intermediate, data, request_info} so orchestration-internal events (group_chat, handoff_sent, magentic_orchestrator) stay inside the workflow instead of leaking into agent responses via str(data) coercion. - Stop raising on intermediate AgentResponseUpdate in non-streaming run(); surface the partial as a Message with text_reasoning content. The defensive raise still applies to terminal output events, where Update payloads would corrupt message ordering. - Extend the DevUI workflow-event mapper so intermediate yields wrapping plain strings, Messages, and list[Message] render as visible output items instead of generic completed-trace events. - Add orchestration coverage for GroupChat, Handoff, and Magentic builders (default vs intermediate_outputs=True; structural where end-to-end is heavy). * Lift output-designation policy into a value type Replace the ``Workflow._output_executors`` list and the ``RunnerContext.should_label_as_intermediate`` Protocol method with a single immutable ``OutputDesignation`` value type owned by ``Workflow``. Thread the designation as a parameter through the existing call chain (Runner -> EdgeRunner -> Executor -> WorkflowContext) so ``yield_output`` consults the threaded snapshot directly rather than calling back into the runner context. Removes the ``InProcRunnerContext._workflow`` back-reference and the ``WorkflowBuilder.build()`` assignment that wired it up. Adds the public predicate ``Workflow.is_terminal_executor(executor_id)`` for external observers; ``OutputDesignation`` itself stays package-internal. Key decisions - ``OutputDesignation.designated`` is ``frozenset[str] | None`` -- ``None`` preserves legacy "every yield is type='output'" behavior, any frozenset (including empty) opts into strict mode. The ``DeprecationWarning`` for legacy mode at build time is unchanged. - ``output_designation`` is an optional parameter on ``Runner``, ``EdgeRunner.send_message``, ``EdgeRunner._execute_on_target``, ``Executor.execute``, ``Executor._create_context_for_handler``, and ``WorkflowContext.__init__``. Each defaults to legacy ``OutputDesignation()`` so direct callers (Azure Functions ``CapturingRunnerContext``, ``test_runner`` recording fixtures) keep working without ceremony. - The workflow-level filter in ``_run_core`` reads ``self._output_designation`` live, preserving today's semantics where mutating the designation after build still affects subsequent runs (used by two existing tests). - ``Workflow.to_dict()`` continues to emit ``"output_executors": list[str] | None`` (sorted from the frozenset). Checkpoint format unchanged. Files changed - _workflow.py: add ``OutputDesignation`` dataclass; replace ``_output_executors`` with ``_output_designation``; add ``is_terminal_executor``; delete ``_should_yield_output_event``. - _runner_context.py: drop ``should_label_as_intermediate`` Protocol method and ``InProcRunnerContext`` impl; drop ``_workflow`` back-reference. - _workflow_builder.py: remove ``context._workflow = workflow`` assignment. - _runner.py, _edge_runner.py, _executor.py, _workflow_context.py: thread ``output_designation`` parameter through the call chain. - tests/workflow/test_output_designation.py (new): three-state coverage of the value type plus the public predicate delegation. - tests/workflow/test_workflow_builder.py, test_validation.py, test_workflow.py, test_runner.py and orchestrations/tests/test_orchestration_intermediate_vs_terminal.py: switch probes from ``_output_executors`` set checks to ``get_output_executors`` / ``is_terminal_executor``; update two post-build mutation tests to set ``_output_designation`` instead. Verification - core/tests/workflow/, orchestrations/tests/, azurefunctions/tests/: 1119 passed, 42 skipped, 2 xfailed. - ``uv run poe lint``: clean. - ``uv run poe typing``: only the pre-existing ``_AGENT_FORWARDED_EVENT_TYPES`` pyright warning from 394bcd607 remains. Notes for next iteration - The builder's own ``_output_executors`` attribute (``list[Executor | SupportsAgentRun]``) is intentionally untouched; the issue scoped the rename to the workflow attribute. - Adjacent review candidates (twin ``WorkflowAgent`` translators, ``_AGENT_FORWARDED_EVENT_TYPES`` kind classifier, ``_event_origin_context`` ContextVar removal, ``WorkflowEvent`` ADT split, legacy-mode removal) remain out of scope. * Add explicit workflow output designation Key decisions - Extend the internal OutputDesignation value type from terminal-only membership to output/intermediate/hidden classification. Legacy mode remains outputs=None, so workflows built without output_executors or intermediate_executors still label every yield_output as type='output'. - WorkflowBuilder now accepts intermediate_executors. Providing either designation enters explicit mode; output executors emit output, intermediate executors emit intermediate, and unlisted yield_output payloads are hidden from caller-facing events while remaining in executor_completed data. - Empty explicit designation, duplicate entries, overlaps, unknown executors, and designated executors without workflow output annotations fail build validation. Existing orchestration builders pass intermediate-capable participants through intermediate_executors to preserve current intermediate_outputs behavior until participant-oriented designation lands. Files changed - packages/core/agent_framework/_workflows/_workflow.py, _workflow_builder.py, _workflow_context.py, _validation.py, _events.py - packages/core/tests/workflow/test_output_designation.py, test_output_executors_contract.py, test_strict_mode_event_labeling.py, test_validation.py, test_workflow.py, test_workflow_agent_intermediate.py - packages/orchestrations/agent_framework_orchestrations/_sequential.py, _concurrent.py, _group_chat.py, _magentic.py - packages/core/AGENTS.md Verification - uv run pytest packages/core/tests/workflow packages/orchestrations/tests packages/devui/tests/devui/test_mapper.py -q - uv run pytest packages/azurefunctions/tests -q - uv run poe lint - uv run poe typing fails only on pre-existing packages/core/agent_framework/_workflows/_agent.py _AGENT_FORWARDED_EVENT_TYPES private-use pyright error. Notes for next iteration - issues/03-core-workflow-explicit-designation.md was moved to issues/done but issues/ remains untracked and intentionally excluded from this commit. - Slice 4 should tighten workflow.as_agent() mapping for hidden emissions and streaming-only update payloads; Slice 5 should replace orchestration intermediate_outputs with participant-oriented designation. * Tighten workflow-as-agent output mapping Key decisions - Treat AgentResponseUpdate as a streaming-only payload across the workflow.as_agent() adapter, so non-streaming agent runs now reject both terminal output and intermediate workflow events carrying updates. - Keep streaming classification behavior explicit: terminal update payloads remain normal text content, while intermediate update payloads are rewritten to text_reasoning content. - Add explicit-mode coverage proving hidden yield_output emissions do not appear in non-streaming AgentResponse messages or streaming AgentResponseUpdate chunks. Files changed - packages/core/agent_framework/_workflows/_agent.py - packages/core/tests/workflow/test_workflow_agent_intermediate.py Verification - uv run pytest packages/core/tests/workflow/test_workflow_agent_intermediate.py -q - uv run pytest packages/core/tests/workflow/test_workflow_agent.py packages/core/tests/workflow/test_workflow_agent_intermediate.py -q - uv run pytest packages/core/tests/workflow packages/orchestrations/tests packages/devui/tests/devui/test_mapper.py -q - uv run poe lint - uv run poe typing fails only on the pre-existing packages/core/agent_framework/_workflows/_agent.py _AGENT_FORWARDED_EVENT_TYPES private-use pyright error. Blockers or notes for next iteration - issues/04-workflow-as-agent-output-mapping.md was moved to issues/done/ but issues/ remains untracked and intentionally excluded from this commit. - Slice 5 should replace orchestration intermediate_outputs with participant-oriented designation. * Add orchestration participant output designation Key decisions - Replace orchestration intermediate_outputs with participant-oriented output_participants and intermediate_participants across Sequential, Concurrent, GroupChat, Magentic, and Handoff builders. - Keep synthetic final executors terminal by default for Concurrent, GroupChat, and Magentic; keep Sequential's final participant terminal by default; keep Handoff participants terminal by default. - Centralize participant designation validation for empty explicit designation, duplicates, overlaps, and unknown participants, then map validated participants to workflow output/intermediate executors. Files changed - packages/orchestrations/agent_framework_orchestrations/_participant_designation.py - packages/orchestrations/agent_framework_orchestrations/_sequential.py - packages/orchestrations/agent_framework_orchestrations/_concurrent.py - packages/orchestrations/agent_framework_orchestrations/_group_chat.py - packages/orchestrations/agent_framework_orchestrations/_magentic.py - packages/orchestrations/agent_framework_orchestrations/_handoff.py - packages/orchestrations/tests/test_orchestration_intermediate_vs_terminal.py - packages/orchestrations/tests/test_magentic.py Blockers or notes for next iteration - issues/05-orchestration-participant-designation.md was moved to issues/done/ but issues/ remains untracked and intentionally excluded from this commit. - Slice 7 should migrate samples and docs away from intermediate_outputs to the new participant designation API. - uv run poe typing still fails only on the pre-existing packages/core/agent_framework/_workflows/_agent.py _AGENT_FORWARDED_EVENT_TYPES private-use pyright error. * Migrate samples to explicit output designation Key decisions - Replace sample usage of the removed orchestration intermediate_outputs boolean with participant-oriented intermediate_participants designation. - Update raw workflow guidance to show output_executors together with intermediate_executors, and document that unlisted yields are hidden in explicit designation mode. - Keep orchestration final outputs terminal while streaming designated participant responses as intermediate progress, including workflow.as_agent() samples where intermediates map to text_reasoning content. - Refresh workflow and orchestration README guidance plus the changelog reference so public docs no longer point users at intermediate_outputs. Files changed - CHANGELOG.md - packages/orchestrations/README.md - samples/README.md - samples/03-workflows/README.md - samples/03-workflows/control-flow/intermediate_vs_terminal_outputs.py - samples/03-workflows/orchestrations/README.md - samples/03-workflows/orchestrations/group_chat_agent_manager.py - samples/03-workflows/orchestrations/group_chat_philosophical_debate.py - samples/03-workflows/orchestrations/group_chat_simple_selector.py - samples/03-workflows/orchestrations/magentic.py - samples/03-workflows/orchestrations/magentic_human_plan_review.py - samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py - samples/03-workflows/agents/group_chat_workflow_as_agent.py - samples/03-workflows/agents/magentic_workflow_as_agent.py - samples/03-workflows/agents/sequential_workflow_as_agent.py - samples/semantic-kernel-migration/orchestrations/group_chat.py - samples/semantic-kernel-migration/orchestrations/magentic.py Blockers or notes for next iteration - issues/07-samples-and-docs-explicit-output-designation.md was moved to issues/done/ but issues/ remains untracked and intentionally excluded from this commit. - issues/06-devui-intermediate-event-rendering.md remains present and appears already satisfied by existing DevUI mapper/tests from the prior implementation slice. - PRD-explicit-workflow-output-designation.md remains untracked and intentionally excluded from this commit. * Render DevUI intermediate workflow outputs Key decisions - Preserve workflow output designation metadata on visible DevUI output messages and text deltas so intermediate/data emissions remain distinguishable from terminal output. - Render intermediate workflow message items in the execution timeline using executor metadata, while excluding them from the final workflow result aggregation. - Keep terminal output message rendering unchanged and retain legacy data events on the intermediate compatibility path. Files changed - packages/devui/agent_framework_devui/_mapper.py - packages/devui/frontend/src/components/features/workflow/execution-timeline.tsx - packages/devui/frontend/src/components/features/workflow/workflow-view.tsx - packages/devui/frontend/src/types/openai.ts - packages/devui/tests/devui/test_mapper.py Blockers or notes for next iteration - issues/06-devui-intermediate-event-rendering.md was moved to issues/done/ but issues/ remains untracked and intentionally excluded from this commit. - PRD-explicit-workflow-output-designation.md remains untracked and intentionally excluded from this commit. - uv run poe typing still fails only on the pre-existing packages/core/agent_framework/_workflows/_agent.py _AGENT_FORWARDED_EVENT_TYPES private-use pyright error. * Fix mypy * Clarify orchestration participant output config * Rename participant output kwargs for clarity output_participants -> final_output_from, intermediate_participants -> intermediate_output_from. The old names read like categories of participant; the new names make it clear the kwarg designates which participants' outputs surface as final vs. intermediate events. * Rename core workflow output kwargs with deprecation shim Adds final_output_from / intermediate_output_from as canonical kwargs on Workflow and WorkflowBuilder. Old output_executors / intermediate_executors kwargs continue to work but emit DeprecationWarning via a shared coalesce helper that also rejects supplying both. Wire-format keys in to_dict() stay as output_executors / intermediate_executors so checkpoint compatibility is preserved. Internal call sites in orchestrations and samples updated to the new names so users following sample code learn the canonical vocabulary; legacy callers still work with a one-shot warning. * Suppress pyright reportPrivateUsage on cross-module sentinel import * Update docstrings * Propagate sub-workflow intermediate outputs, fix handoff/sequential intermediate-only designation, and shore up tests, sample, and docstrings around the intermediate output contract. * Add canonical workflow output_from selection Key decisions:\n- Make output_from the canonical workflow-output allow-list and keep output_executors/final_output_from as deprecated compatibility aliases.\n- Treat empty output_from/intermediate_output_from lists as explicit selections and keep validation responsible for empty, duplicate, overlap, and unknown selections.\n- Remove the branch-only public intermediate_executors WorkflowBuilder kwarg while preserving legacy wire keys in to_dict().\n\nFiles changed:\n- packages/core/agent_framework/_workflows/_workflow.py\n- packages/core/agent_framework/_workflows/_workflow_builder.py\n- packages/core/agent_framework/_workflows/_workflow_context.py\n- packages/core/agent_framework/_workflows/_agent.py\n- packages/core/agent_framework/_workflows/_agent_executor.py\n- packages/core/tests/workflow/* output-selection coverage updates\n- packages/core/AGENTS.md\n- issues/done/001-canonical-list-based-output-selection.md\n\nBlockers/notes:\n- Orchestration builders still pass final_output_from internally; follow-up issue 004 should migrate them to output_from.\n- Legacy omitted-selection behavior and explicit all/all_other literals are left for issues 002 and 003. * Add explicit all workflow output selection Key decisions: - Treat output_from='all' as an explicit workflow-output selection sentinel and expand it at build time to executors with declared workflow output types. - Keep omitted output selections in legacy all-output mode with a deprecation warning that names output_from and intermediate_output_from and points to output_from='all'. - Reject intermediate_output_from='all' at construction because the all-output literal is output-only for this issue. Files changed: - packages/core/agent_framework/_workflows/_workflow_builder.py - packages/core/tests/workflow/test_output_executors_contract.py - issues/done/002-explicit-all-output-and-legacy-migration.md Blockers/notes: - all_other intermediate-output selection remains for issue 003. - Workflow-as-agent/orchestration parity remains for issue 004. * Add all-other intermediate output selection Key decisions: - Treat intermediate_output_from='all_other' as an explicit intermediate-output selection sentinel and expand it at build time after the workflow graph is complete. - Expand all_other to output-capable executors not selected by output_from; omitted or empty output_from selects no workflow outputs, while output_from='all' leaves an empty intermediate selection. - Keep output_from='all_other' invalid so all_other remains intermediate-output-only and runtime classification still receives concrete executor-id sets. Files changed: - packages/core/agent_framework/_workflows/_workflow_builder.py - packages/core/tests/workflow/test_output_executors_contract.py - issues/done/003-all-other-intermediate-output-selection.md Blockers/notes: - Workflow-as-agent and orchestration parity remains for issue 004. - Full documentation updates remain for issue 005. * Add orchestration output selection parity Key decisions: - Expose output_from on sequential, concurrent, group chat, handoff, and magentic builders while keeping final_output_from as a deprecated compatibility alias. - Resolve orchestration participant selections through the same explicit rules as workflows: output_from='all', intermediate_output_from='all_other', hidden unselected participant payloads, and overlap/duplicate/unknown/invalid-literal validation. - Continue preserving documented orchestration defaults by always designating each pattern's terminal internal executor where applicable. Files changed: - packages/orchestrations/agent_framework_orchestrations/_participant_output_config.py - packages/orchestrations/agent_framework_orchestrations/_sequential.py - packages/orchestrations/agent_framework_orchestrations/_concurrent.py - packages/orchestrations/agent_framework_orchestrations/_group_chat.py - packages/orchestrations/agent_framework_orchestrations/_handoff.py - packages/orchestrations/agent_framework_orchestrations/_magentic.py - packages/orchestrations/agent_framework_orchestrations/_orchestration_request_info.py - packages/orchestrations/tests/test_orchestration_intermediate_vs_terminal.py - issues/done/004-workflow-as-agent-and-orchestration-parity.md Blockers/notes: - Full documentation and sample migration wording remains for issue 005. - Existing tests that intentionally use final_output_from now emit the new deprecation warning. * Document workflow output selection contract Key decisions: - Use Workflow Output and Intermediate Output as the developer-facing terms for selected caller-facing emissions. - Document output_from and intermediate_output_from as the canonical API, with output_from as an allow-list and unselected payloads hidden unless explicitly selected as intermediate. - Add scenario and invalid-selection tables for workflow and orchestration docs, including legacy omission warnings, output_from='all', intermediate_output_from='all_other', list selections, invalid literals, overlap, duplicates, unknown selections, and empty explicit selections. - Migrate samples away from final_output_from and output_executors except where compatibility aliases are explicitly documented. Files changed: - packages/core/AGENTS.md - packages/orchestrations/README.md - packages/orchestrations/agent_framework_orchestrations/_handoff.py - packages/orchestrations/agent_framework_orchestrations/_sequential.py - samples/03-workflows/README.md - samples/03-workflows/control-flow/intermediate_vs_terminal_outputs.py - samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py - samples/03-workflows/orchestrations/README.md - samples/04-hosting/foundry-hosted-agents/responses/05_workflows/main.py - scripts/sample_validation/create_dynamic_workflow_executor.py - issues/done/005-document-output-selection-contract.md Blockers/notes: - Direct full Ruff on scripts/sample_validation/create_dynamic_workflow_executor.py still reports pre-existing docstring/print/line-length issues outside this docs migration; syntax-focused checks for changed files pass. - No remaining AFK issue files are present under issues/. * Latest updates * Typing fixes * Cleanup --- python/CHANGELOG.md | 2 +- .../agent_framework_azurefunctions/_app.py | 13 + .../_context.py | 10 + .../azurefunctions/tests/test_func_utils.py | 10 +- python/packages/core/AGENTS.md | 9 +- .../core/agent_framework/_workflows/_agent.py | 47 +- .../_workflows/_agent_executor.py | 4 +- .../_workflows/_edge_runner.py | 40 +- .../agent_framework/_workflows/_events.py | 52 +- .../agent_framework/_workflows/_functional.py | 3 +- .../_workflows/_runner_context.py | 22 +- .../agent_framework/_workflows/_validation.py | 32 +- .../agent_framework/_workflows/_workflow.py | 202 ++++- .../_workflows/_workflow_builder.py | 230 +++++- .../_workflows/_workflow_context.py | 30 +- .../_workflows/_workflow_executor.py | 18 +- .../test_agent_executor_tool_calls.py | 16 +- .../workflow/test_agent_run_event_typing.py | 6 +- .../tests/workflow/test_full_conversation.py | 10 +- .../workflow/test_functional_workflow.py | 8 +- .../tests/workflow/test_output_designation.py | 137 ++++ .../test_output_executors_contract.py | 287 +++++++ .../core/tests/workflow/test_runner.py | 16 +- .../core/tests/workflow/test_serialization.py | 45 ++ .../test_strict_mode_event_labeling.py | 118 +++ .../core/tests/workflow/test_sub_workflow.py | 72 ++ .../core/tests/workflow/test_validation.py | 83 +- .../core/tests/workflow/test_workflow.py | 30 +- .../tests/workflow/test_workflow_agent.py | 2 +- .../test_workflow_agent_intermediate.py | 353 +++++++++ .../tests/workflow/test_workflow_builder.py | 48 +- .../tests/workflow/test_workflow_context.py | 26 + .../workflow/test_workflow_event_factories.py | 34 + .../tests/workflow/test_workflow_kwargs.py | 2 +- .../devui/agent_framework_devui/_mapper.py | 78 +- .../features/workflow/execution-timeline.tsx | 45 +- .../features/workflow/workflow-view.tsx | 42 +- .../devui/frontend/src/types/openai.ts | 2 + .../packages/devui/tests/devui/test_mapper.py | 113 ++- .../foundry/tests/test_foundry_evals.py | 12 +- python/packages/orchestrations/README.md | 45 ++ .../_concurrent.py | 36 +- .../_group_chat.py | 38 +- .../_handoff.py | 35 +- .../_magentic.py | 38 +- .../_orchestration_request_info.py | 8 +- .../_participant_output_config.py | 166 ++++ .../_sequential.py | 41 +- .../orchestrations/tests/test_magentic.py | 9 +- ..._orchestration_intermediate_vs_terminal.py | 749 ++++++++++++++++++ python/samples/03-workflows/README.md | 40 +- .../agents/group_chat_workflow_as_agent.py | 6 +- .../agents/magentic_workflow_as_agent.py | 6 +- .../agents/sequential_workflow_as_agent.py | 6 +- .../intermediate_vs_terminal_outputs.py | 156 ++++ .../agents_with_approval_requests.py | 2 +- .../03-workflows/orchestrations/README.md | 37 +- .../group_chat_agent_manager.py | 9 +- .../group_chat_philosophical_debate.py | 11 +- .../group_chat_simple_selector.py | 9 +- .../03-workflows/orchestrations/magentic.py | 9 +- .../magentic_human_plan_review.py | 11 +- .../sequential_chain_only_agent_responses.py | 4 +- .../responses/05_workflows/main.py | 6 +- python/samples/README.md | 2 +- .../orchestrations/group_chat.py | 4 +- .../orchestrations/magentic.py | 4 +- .../create_dynamic_workflow_executor.py | 9 +- 68 files changed, 3480 insertions(+), 325 deletions(-) create mode 100644 python/packages/core/tests/workflow/test_output_designation.py create mode 100644 python/packages/core/tests/workflow/test_output_executors_contract.py create mode 100644 python/packages/core/tests/workflow/test_strict_mode_event_labeling.py create mode 100644 python/packages/core/tests/workflow/test_workflow_agent_intermediate.py create mode 100644 python/packages/core/tests/workflow/test_workflow_event_factories.py create mode 100644 python/packages/orchestrations/agent_framework_orchestrations/_participant_output_config.py create mode 100644 python/packages/orchestrations/tests/test_orchestration_intermediate_vs_terminal.py create mode 100644 python/samples/03-workflows/control-flow/intermediate_vs_terminal_outputs.py diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index e2adf2762e..e800652483 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -67,7 +67,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **agent-framework-foundry-hosting**: Add hosted Durable Workflow support — propagate full conversation history to workflow agents and wire `Workflow.as_agent()` end-to-end via the foundry hosting layer ([#5531](https://github.com/microsoft/agent-framework/pull/5531)) ### Changed -- **agent-framework-orchestrations**: [BREAKING] Standardize orchestration terminal outputs as `AgentResponse` so `Workflow.as_agent()` returns the final answer only; aligns sequential-approval (`with_request_info`) and concurrent (`intermediate_outputs=True`) flows on the same output contract ([#5301](https://github.com/microsoft/agent-framework/pull/5301)) +- **agent-framework-orchestrations**: [BREAKING] Standardize orchestration terminal outputs as `AgentResponse` so `Workflow.as_agent()` returns the final answer only; aligns sequential-approval (`with_request_info`) and concurrent participant output designation flows on the same output contract ([#5301](https://github.com/microsoft/agent-framework/pull/5301)) - **agent-framework-core**, **agent-framework-declarative**: Preserve `Workflow.run()` shared state across calls so multi-turn `WorkflowAgent` invocations retain context, accept `list[Message]` input in the declarative start executor, and coerce `Enum` values when serializing PowerFx symbols ([#5531](https://github.com/microsoft/agent-framework/pull/5531)) - **dependencies**: Update workspace package dependencies and preserve `mcp[ws]` / `uvicorn[standard]` extras through override-dependencies in `/python` ([#5555](https://github.com/microsoft/agent-framework/pull/5555)) diff --git a/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py b/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py index e1164154a5..c25c2461ce 100644 --- a/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py +++ b/python/packages/azurefunctions/agent_framework_azurefunctions/_app.py @@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, TypeVar, cast import azure.durable_functions as df import azure.functions as func from agent_framework import AgentExecutor, SupportsAgentRun, Workflow, WorkflowEvent +from agent_framework._workflows._runner_context import YieldOutputEventType from agent_framework_durabletask import ( DEFAULT_MAX_POLL_RETRIES, DEFAULT_POLL_INTERVAL_SECONDS, @@ -307,6 +308,18 @@ class AgentFunctionApp(DFAppBase): async def run() -> dict[str, Any]: # Create runner context and shared state runner_context = CapturingRunnerContext() + workflow = self.workflow + + def classify_yielded_output(executor_id: str) -> YieldOutputEventType | None: + if workflow is None: + return "output" + if workflow.is_terminal_executor(executor_id): + return "output" + if workflow.is_intermediate_executor(executor_id): + return "intermediate" + return None + + runner_context.set_yield_output_classifier(classify_yielded_output) shared_state = State() # Deserialize shared state values to reconstruct dataclasses/Pydantic models diff --git a/python/packages/azurefunctions/agent_framework_azurefunctions/_context.py b/python/packages/azurefunctions/agent_framework_azurefunctions/_context.py index a45dcf81fc..4912fe4cc9 100644 --- a/python/packages/azurefunctions/agent_framework_azurefunctions/_context.py +++ b/python/packages/azurefunctions/agent_framework_azurefunctions/_context.py @@ -19,6 +19,7 @@ from agent_framework import ( WorkflowEvent, WorkflowMessage, ) +from agent_framework._workflows._runner_context import YieldOutputClassifier, YieldOutputEventType from agent_framework._workflows._state import State @@ -41,6 +42,7 @@ class CapturingRunnerContext(RunnerContext): self._pending_request_info_events: dict[str, WorkflowEvent[Any]] = {} self._workflow_id: str | None = None self._streaming: bool = False + self._yield_output_classifier: YieldOutputClassifier = lambda _executor_id: "output" # region Messaging @@ -144,6 +146,14 @@ class CapturingRunnerContext(RunnerContext): """Check if streaming mode is enabled (always False in activity context).""" return self._streaming + def set_yield_output_classifier(self, classifier: YieldOutputClassifier) -> None: + """Set the classifier used by WorkflowContext.yield_output().""" + self._yield_output_classifier = classifier + + def classify_yielded_output(self, executor_id: str) -> YieldOutputEventType | None: + """Classify an executor's yield_output payload as output, intermediate, or hidden.""" + return self._yield_output_classifier(executor_id) + # endregion Workflow Configuration # region Request Info Events diff --git a/python/packages/azurefunctions/tests/test_func_utils.py b/python/packages/azurefunctions/tests/test_func_utils.py index 6110c0f895..30841eaec0 100644 --- a/python/packages/azurefunctions/tests/test_func_utils.py +++ b/python/packages/azurefunctions/tests/test_func_utils.py @@ -107,7 +107,7 @@ class TestCapturingRunnerContext: @pytest.mark.asyncio async def test_add_event_queues_event(self, context: CapturingRunnerContext) -> None: """Test that add_event queues events correctly.""" - event = WorkflowEvent.output(executor_id="exec_1", data="output") + event = WorkflowEvent("output", executor_id="exec_1", data="output") await context.add_event(event) @@ -120,7 +120,7 @@ class TestCapturingRunnerContext: @pytest.mark.asyncio async def test_drain_events_clears_queue(self, context: CapturingRunnerContext) -> None: """Test that drain_events clears the event queue.""" - await context.add_event(WorkflowEvent.output(executor_id="e", data="test")) + await context.add_event(WorkflowEvent("output", executor_id="e", data="test")) await context.drain_events() # First drain events = await context.drain_events() # Second drain @@ -132,14 +132,14 @@ class TestCapturingRunnerContext: """Test has_events returns correct boolean.""" assert await context.has_events() is False - await context.add_event(WorkflowEvent.output(executor_id="e", data="test")) + await context.add_event(WorkflowEvent("output", executor_id="e", data="test")) assert await context.has_events() is True @pytest.mark.asyncio async def test_next_event_waits_for_event(self, context: CapturingRunnerContext) -> None: """Test that next_event returns queued events.""" - event = WorkflowEvent.output(executor_id="e", data="waited") + event = WorkflowEvent("output", executor_id="e", data="waited") await context.add_event(event) result = await context.next_event() @@ -171,7 +171,7 @@ class TestCapturingRunnerContext: async def test_reset_for_new_run_clears_state(self, context: CapturingRunnerContext) -> None: """Test that reset_for_new_run clears all state.""" await context.send_message(WorkflowMessage(data="test", target_id="t", source_id="s")) - await context.add_event(WorkflowEvent.output(executor_id="e", data="event")) + await context.add_event(WorkflowEvent("output", executor_id="e", data="event")) context.set_streaming(True) context.reset_for_new_run() diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index edd4eaa158..ed47f363a2 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -79,7 +79,14 @@ agent_framework/ ### Workflows (`_workflows/`) - **`Workflow`** - Graph-based workflow definition -- **`WorkflowBuilder`** - Fluent API for building workflows +- **`WorkflowBuilder`** - Fluent API for building workflows, including explicit + `output_from` / `intermediate_output_from` selection for caller-facing emissions. `output_from` + is an allow-list for **Workflow Output**; unselected executor payloads are hidden unless + `intermediate_output_from` selects them as **Intermediate Output**. Use `output_from="all"` for + explicit all-output behavior and `intermediate_output_from="all_other"` for visible progress from + every output-capable executor not selected by `output_from`. +- **`WorkflowRunResult`** - Non-streaming workflow result with Workflow Output `get_outputs()` + and Intermediate Output `get_intermediate_outputs()` accessors - **Orchestrators**: `SequentialOrchestrator`, `ConcurrentOrchestrator`, `GroupChatOrchestrator`, `MagenticOrchestrator`, `HandoffOrchestrator` ## Built-in Providers diff --git a/python/packages/core/agent_framework/_workflows/_agent.py b/python/packages/core/agent_framework/_workflows/_agent.py index f8a85a261c..2d9b37e1f5 100644 --- a/python/packages/core/agent_framework/_workflows/_agent.py +++ b/python/packages/core/agent_framework/_workflows/_agent.py @@ -32,6 +32,7 @@ from .._types import ( from ..exceptions import AgentInvalidRequestException, AgentInvalidResponseException from ._checkpoint import CheckpointStorage from ._events import ( + AGENT_FORWARDED_EVENT_TYPES, WorkflowEvent, ) from ._message_utils import normalize_messages_input @@ -104,7 +105,7 @@ class WorkflowAgent(BaseAgent): Note: Only output events (type='output') and request_info events (type='request_info') from the workflow are considered and converted to agent responses of the WorkflowAgent. - Other workflow events are ignored. Use `with_output_from` in WorkflowBuilder to control + Other workflow events are ignored. Use `output_from` in WorkflowBuilder to control which executors' outputs are surfaced as agent responses. """ if id is None: @@ -300,7 +301,7 @@ class WorkflowAgent(BaseAgent): function_invocation_kwargs=function_invocation_kwargs, client_kwargs=client_kwargs, ): - if event.type == "output" or event.type == "request_info": + if event.type in AGENT_FORWARDED_EVENT_TYPES: output_events.append(event) result = self._convert_workflow_events_to_agent_response(response_id, output_events) @@ -514,7 +515,11 @@ class WorkflowAgent(BaseAgent): response_id: str, output_events: list[WorkflowEvent[Any]], ) -> AgentResponse: - """Convert a list of workflow output events to an AgentResponse.""" + """Convert a list of workflow events to an AgentResponse. + + Caller-facing workflow events are forwarded as agent messages. Terminal and + intermediate event payloads keep their original content types. + """ messages: list[Message] = [] raw_representations: list[object] = [] merged_usage: UsageDetails | None = None @@ -535,14 +540,19 @@ class WorkflowAgent(BaseAgent): raw_representations.append(output_event) else: data = output_event.data + # Anything that isn't `output` is intermediate — this branch only sees + # events that already passed the lifecycle filter and weren't request_info. + is_intermediate = output_event.type != "output" if isinstance(data, AgentResponseUpdate): - # We cannot support AgentResponseUpdate in non-streaming mode. This is because the message - # sequence cannot be guaranteed when there are streaming updates in between non-streaming - # responses. + # AgentResponseUpdate is a streaming-only payload. Accepting it + # in non-streaming runs would make message ordering depend on + # partial chunks for both terminal and intermediate events. + event_label = "Intermediate" if is_intermediate else "Output" raise AgentInvalidRequestException( - "Output event with AgentResponseUpdate data cannot be emitted in non-streaming mode. " - "Please ensure executors emit AgentResponse for non-streaming workflows." + f"{event_label} event with AgentResponseUpdate data cannot be emitted " + "in non-streaming mode. Please ensure executors emit AgentResponse " + "for non-streaming workflows." ) if isinstance(data, AgentResponse): @@ -626,16 +636,21 @@ class WorkflowAgent(BaseAgent): ) -> list[AgentResponseUpdate]: """Convert a workflow event to a list of AgentResponseUpdate objects. - Events with type='output' and type='request_info' are processed. - Other workflow events are ignored as they are workflow-internal. + Forwarding rule: - For 'output' events, AgentExecutor yields AgentResponseUpdate for streaming updates - via ctx.yield_output(). This method converts those to agent response updates. - - Returns: - A list of AgentResponseUpdate objects. Empty list if the event is not relevant. + - ``type='output'`` — terminal user-facing emission. Forwarded as-is. + - ``type='intermediate'`` (and the deprecated ``type='data'``) — forwarded + as-is. + - ``type='request_info'`` — request-info translation (unchanged). + - Everything else (lifecycle, diagnostics, executor bookkeeping, + orchestration-internal events like ``group_chat``/``handoff_sent``/ + ``magentic_orchestrator``) is dropped. """ - if event.type == "output": + # TODO(evmattso): https://github.com/microsoft/agent-framework/issues/5885 + if event.type not in AGENT_FORWARDED_EVENT_TYPES: + return [] + + if event.type != "request_info": data = event.data executor_id = event.executor_id diff --git a/python/packages/core/agent_framework/_workflows/_agent_executor.py b/python/packages/core/agent_framework/_workflows/_agent_executor.py index 9b16b1f291..16e4fd3def 100644 --- a/python/packages/core/agent_framework/_workflows/_agent_executor.py +++ b/python/packages/core/agent_framework/_workflows/_agent_executor.py @@ -123,7 +123,7 @@ class AgentExecutor(Executor): - run(stream=True): Emits incremental output events (type='output') as the agent produces tokens - run(): Emits a single output event (type='output') containing the complete response - Use `with_output_from` in WorkflowBuilder to control whether the AgentResponse + Use `output_from` in WorkflowBuilder to control whether the AgentResponse or AgentResponseUpdate objects are yielded as workflow outputs. Messages sent to downstream executors will always be the complete AgentResponse. In @@ -478,7 +478,7 @@ class AgentExecutor(Executor): # Prefer stream finalization when available so result hooks run # (e.g., thread conversation updates). Fall back to reconstructing from updates - # for legacy/custom agents that return a plain async iterable. + # for compatibility/custom agents that return a plain async iterable. # TODO(evmattso): Integrate workflow agent run handling around ResponseStream so # AgentExecutor does not need this conditional stream-finalization branch. maybe_get_final_response = getattr(stream, "get_final_response", None) diff --git a/python/packages/core/agent_framework/_workflows/_edge_runner.py b/python/packages/core/agent_framework/_workflows/_edge_runner.py index 06188e9fb2..586ccd9c3a 100644 --- a/python/packages/core/agent_framework/_workflows/_edge_runner.py +++ b/python/packages/core/agent_framework/_workflows/_edge_runner.py @@ -38,7 +38,12 @@ class EdgeRunner(ABC): self._executors = executors @abstractmethod - async def send_message(self, message: WorkflowMessage, state: State, ctx: RunnerContext) -> bool: + async def send_message( + self, + message: WorkflowMessage, + state: State, + ctx: RunnerContext, + ) -> bool: """Send a message through the edge group. Args: @@ -90,7 +95,12 @@ class SingleEdgeRunner(EdgeRunner): super().__init__(edge_group, executors) self._edge = edge_group.edges[0] - async def send_message(self, message: WorkflowMessage, state: State, ctx: RunnerContext) -> bool: + async def send_message( + self, + message: WorkflowMessage, + state: State, + ctx: RunnerContext, + ) -> bool: """Send a message through the single edge.""" should_execute = False target_id: str | None = None @@ -162,7 +172,12 @@ class FanOutEdgeRunner(EdgeRunner): Callable[[Any, list[str]], list[str]] | None, getattr(edge_group, "selection_func", None) ) - async def send_message(self, message: WorkflowMessage, state: State, ctx: RunnerContext) -> bool: + async def send_message( + self, + message: WorkflowMessage, + state: State, + ctx: RunnerContext, + ) -> bool: """Send a message through all edges in the fan-out edge group.""" deliverable_edges: list[Edge] = [] single_target_edge: Edge | None = None @@ -253,7 +268,11 @@ class FanOutEdgeRunner(EdgeRunner): # Execute outside the span if single_target_edge: await self._execute_on_target( - single_target_edge.target_id, [single_target_edge.source_id], message, state, ctx + single_target_edge.target_id, + [single_target_edge.source_id], + message, + state, + ctx, ) return True @@ -285,7 +304,12 @@ class FanInEdgeRunner(EdgeRunner): # Key is the source executor ID, value is a list of messages self._buffer: dict[str, list[WorkflowMessage]] = defaultdict(list) - async def send_message(self, message: WorkflowMessage, state: State, ctx: RunnerContext) -> bool: + async def send_message( + self, + message: WorkflowMessage, + state: State, + ctx: RunnerContext, + ) -> bool: """Send a message through all edges in the fan-in edge group.""" execution_data: dict[str, Any] | None = None with create_edge_group_processing_span( @@ -362,7 +386,11 @@ class FanInEdgeRunner(EdgeRunner): # Execute outside the span if needed if execution_data: await self._execute_on_target( - execution_data["target_id"], execution_data["source_ids"], execution_data["message"], state, ctx + execution_data["target_id"], + execution_data["source_ids"], + execution_data["message"], + state, + ctx, ) return True diff --git a/python/packages/core/agent_framework/_workflows/_events.py b/python/packages/core/agent_framework/_workflows/_events.py index 4b8238268c..aa1a69954f 100644 --- a/python/packages/core/agent_framework/_workflows/_events.py +++ b/python/packages/core/agent_framework/_workflows/_events.py @@ -5,6 +5,7 @@ from __future__ import annotations import builtins import sys import traceback as _traceback +import warnings from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar @@ -106,8 +107,9 @@ WorkflowEventType = Literal[ "status", # Workflow state changed (use .state) "failed", # Workflow terminated with error (use .details) # Data events - "output", # Executor yielded final output (use .executor_id, .data) - "data", # Executor emitted data during execution (use .executor_id, .data) + "output", # Executor yielded final terminal output (use .executor_id, .data) + "intermediate", # Executor emitted intermediate (non-terminal) output (use .executor_id, .data) + "data", # DEPRECATED — compatibility alias for intermediate emissions; use type='intermediate' instead. # Request events (human-in-the-loop) "request_info", # Executor requests external info (use .request_id, .source_executor_id) # Diagnostic events (warnings/errors from user code) @@ -128,21 +130,34 @@ WorkflowEventType = Literal[ ] +# Event types forwarded across the ``workflow.as_agent()`` boundary. Anything not +# in this set — lifecycle events, diagnostics, executor bookkeeping, and +# orchestration-internal events (``group_chat``, ``handoff_sent``, +# ``magentic_orchestrator``) — stays inside the workflow and is not surfaced to +# agent callers. Internal to the ``_workflows`` package. +AGENT_FORWARDED_EVENT_TYPES: frozenset[str] = frozenset({ + "output", + "intermediate", + "data", # deprecated alias for intermediate; retained for backward compat + "request_info", +}) + + class WorkflowEvent(Generic[DataT]): """Unified event for all workflow emissions. This single generic class handles all workflow events through a `type` discriminator, following the same pattern as the `Content` class. - Use factory methods for convenient construction: + Use factory methods for convenient construction of lifecycle, diagnostic, request, + and executor bookkeeping events. Workflow ``output`` and ``intermediate`` events + are emitted by ``ctx.yield_output(...)`` based on workflow output selection. - `WorkflowEvent.started()` - workflow run began - `WorkflowEvent.status(state)` - workflow state changed - `WorkflowEvent.failed(details)` - workflow terminated with error - `WorkflowEvent.warning(message)` - warning from user code - `WorkflowEvent.error(exception)` - error from user code - - `WorkflowEvent.output(executor_id, data)` - executor yielded final output - - `WorkflowEvent.data(executor_id, data)` - executor emitted data (e.g., AgentResponse) - `WorkflowEvent.request_info(...)` - executor requests external info - `WorkflowEvent.superstep_started(iteration)` - superstep began - `WorkflowEvent.superstep_completed(iteration)` - superstep ended @@ -158,14 +173,13 @@ class WorkflowEvent(Generic[DataT]): Examples: .. code-block:: python - # Create events via factory methods + # Create lifecycle events via factory methods started = WorkflowEvent.started() status = WorkflowEvent.status(WorkflowRunState.IN_PROGRESS) - output = WorkflowEvent.output("agent1", result_data) - # Emit typed data from executor - event: WorkflowEvent[AgentResponse] = WorkflowEvent.data("agent1", response) - data: AgentResponse = event.data # Type-safe access + # Type-safe access to event data + event: WorkflowEvent[AgentResponse] = WorkflowEvent("data", executor_id="agent1", data=response) + data: AgentResponse = event.data # Check event type if event.type == "status": @@ -263,18 +277,20 @@ class WorkflowEvent(Generic[DataT]): """Create an 'error' event from user code.""" return WorkflowEvent("error", data=exception) - @classmethod - def output(cls, executor_id: str, data: DataT) -> WorkflowEvent[DataT]: - """Create an 'output' event when an executor yields final output.""" - return cls("output", executor_id=executor_id, data=data) - @classmethod def emit(cls, executor_id: str, data: DataT) -> WorkflowEvent[DataT]: - """Create a 'data' event when an executor emits data during execution. + """Create a 'data' event (deprecated alias for intermediate emissions). - This is the primary method for executors to emit typed data - (e.g., AgentResponse, AgentResponseUpdate, custom data). + .. deprecated:: + Use ``ctx.yield_output(...)`` and configure ``intermediate_output_from`` instead. + Will be removed in a future major release along with the ``type='data'`` event variant. """ + warnings.warn( + "WorkflowEvent.emit() / type='data' are deprecated; use ctx.yield_output() from an " + "intermediate-designated executor. Will be removed in a future major release.", + DeprecationWarning, + stacklevel=2, + ) return cls("data", executor_id=executor_id, data=data) @classmethod diff --git a/python/packages/core/agent_framework/_workflows/_functional.py b/python/packages/core/agent_framework/_workflows/_functional.py index 159d75e137..5746c2161c 100644 --- a/python/packages/core/agent_framework/_workflows/_functional.py +++ b/python/packages/core/agent_framework/_workflows/_functional.py @@ -982,7 +982,8 @@ class FunctionalWorkflow: # Emit the return value as the workflow output. if return_value is not None: - await ctx.add_event(WorkflowEvent.output(self.name, return_value)) + with _framework_event_origin(): + await ctx.add_event(WorkflowEvent("output", executor_id=self.name, data=return_value)) # Persist step cache for response-only replay self._last_step_cache = dict(ctx._step_cache) diff --git a/python/packages/core/agent_framework/_workflows/_runner_context.py b/python/packages/core/agent_framework/_workflows/_runner_context.py index e3711ea96f..2e4901f411 100644 --- a/python/packages/core/agent_framework/_workflows/_runner_context.py +++ b/python/packages/core/agent_framework/_workflows/_runner_context.py @@ -4,10 +4,11 @@ from __future__ import annotations import asyncio import logging +from collections.abc import Callable from copy import copy from dataclasses import dataclass from enum import Enum -from typing import Any, Protocol, TypeVar, runtime_checkable +from typing import Any, Literal, Protocol, TypeVar, runtime_checkable from ._checkpoint import CheckpointID, CheckpointStorage, WorkflowCheckpoint from ._const import INTERNAL_SOURCE_ID @@ -18,6 +19,8 @@ from ._typing_utils import is_instance_of logger = logging.getLogger(__name__) T = TypeVar("T") +YieldOutputEventType = Literal["output", "intermediate"] +YieldOutputClassifier = Callable[[str], YieldOutputEventType | None] class MessageType(Enum): @@ -263,6 +266,14 @@ class RunnerContext(Protocol): """ ... + def set_yield_output_classifier(self, classifier: YieldOutputClassifier) -> None: + """Set the classifier used by WorkflowContext.yield_output().""" + ... + + def classify_yielded_output(self, executor_id: str) -> YieldOutputEventType | None: + """Classify an executor's yield_output payload as output, intermediate, or hidden.""" + ... + class InProcRunnerContext: """In-process execution context for local execution and optional checkpointing.""" @@ -286,6 +297,7 @@ class InProcRunnerContext: # Streaming flag - set by workflow's run(..., stream=True) vs run(..., stream=False) self._streaming: bool = False + self._yield_output_classifier: YieldOutputClassifier = lambda _executor_id: "output" # region Messaging and Events async def send_message(self, message: WorkflowMessage) -> None: @@ -480,3 +492,11 @@ class InProcRunnerContext: A dictionary mapping request IDs to their corresponding WorkflowEvent (type='request_info'). """ return dict(self._pending_request_info_events) + + def set_yield_output_classifier(self, classifier: YieldOutputClassifier) -> None: + """Set the classifier used by WorkflowContext.yield_output().""" + self._yield_output_classifier = classifier + + def classify_yielded_output(self, executor_id: str) -> YieldOutputEventType | None: + """Classify an executor's yield_output payload as output, intermediate, or hidden.""" + return self._yield_output_classifier(executor_id) diff --git a/python/packages/core/agent_framework/_workflows/_validation.py b/python/packages/core/agent_framework/_workflows/_validation.py index 990668f340..d8aa4c80f5 100644 --- a/python/packages/core/agent_framework/_workflows/_validation.py +++ b/python/packages/core/agent_framework/_workflows/_validation.py @@ -104,6 +104,7 @@ class WorkflowGraphValidator: executors: dict[str, Executor], start_executor: Executor, output_executors: list[str], + intermediate_executors: list[str] | None = None, ) -> None: """Validate the entire workflow graph. @@ -112,6 +113,7 @@ class WorkflowGraphValidator: executors: Map of executor IDs to executor instances start_executor: The starting executor output_executors: List of output executor IDs + intermediate_executors: List of intermediate executor IDs Raises: WorkflowValidationError: If any validation fails @@ -158,7 +160,7 @@ class WorkflowGraphValidator: self._validate_graph_connectivity(start_executor.id) self._validate_self_loops() self._validate_dead_ends() - self._output_validation(output_executors) + self._output_validation(output_executors, intermediate_executors or []) def _validate_handler_output_annotations(self) -> None: """Validate that each handler's ctx parameter is annotated with WorkflowContext[T]. @@ -356,8 +358,15 @@ class WorkflowGraphValidator: # region Output Validation - def _output_validation(self, output_executors: list[str]) -> None: - """Validate that output executors exist in the workflow and have the correct workflow context annotations.""" + def _output_validation(self, output_executors: list[str], intermediate_executors: list[str]) -> None: + """Validate that designated executors exist and have workflow output annotations.""" + overlap = sorted(set(output_executors).intersection(intermediate_executors)) + if overlap: + raise WorkflowValidationError( + f"Executors cannot be both output and intermediate designated: {overlap}", + validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, + ) + for output_id in output_executors: if output_id not in self._executors: raise WorkflowValidationError( @@ -372,6 +381,20 @@ class WorkflowGraphValidator: validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, ) + for intermediate_id in intermediate_executors: + if intermediate_id not in self._executors: + raise WorkflowValidationError( + f"Intermediate executor '{intermediate_id}' is not present in the workflow graph", + validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, + ) + + intermediate_executor = self._executors[intermediate_id] + if not intermediate_executor.workflow_output_types: + raise WorkflowValidationError( + f"Intermediate executor '{intermediate_id}' must have output type annotations defined.", + validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, + ) + # endregion # region Additional Validation Scenarios @@ -415,6 +438,7 @@ def validate_workflow_graph( executors: dict[str, Executor], start_executor: Executor, output_executors: list[str], + intermediate_executors: list[str] | None = None, ) -> None: """Convenience function to validate a workflow graph. @@ -423,6 +447,7 @@ def validate_workflow_graph( executors: Map of executor IDs to executor instances start_executor: The starting executor instance output_executors: List of output executor IDs + intermediate_executors: List of intermediate executor IDs Raises: WorkflowValidationError: If any validation fails @@ -433,4 +458,5 @@ def validate_workflow_graph( executors, start_executor, output_executors, + intermediate_executors, ) diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index 1c67b8b6f8..0493cd015f 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -10,7 +10,9 @@ import json import logging import types import uuid +import warnings from collections.abc import AsyncIterable, Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal, overload from .._sessions import ContextProvider @@ -34,6 +36,7 @@ from ._runner import Runner from ._runner_context import RunnerContext from ._state import State from ._typing_utils import is_instance_of, try_coerce_to_type +from ._validation import ValidationTypeEnum, WorkflowValidationError if TYPE_CHECKING: from ._agent import WorkflowAgent @@ -41,6 +44,60 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_MISSING: Any = object() + + +def _coalesce_renamed_kwarg(old_name: str, old_value: Any, new_name: str, new_value: Any) -> Any: + """Resolve a renamed keyword argument while keeping the deprecated name working. + + Pass ``_MISSING`` (not ``None``) for the value that was not supplied — ``None`` is + a legitimate user-supplied value for these kwargs. + """ + old_supplied = old_value is not _MISSING + new_supplied = new_value is not _MISSING + if old_supplied and new_supplied: + raise TypeError(f"Cannot pass both `{old_name}` (deprecated) and `{new_name}`; use `{new_name}` only.") + if old_supplied: + warnings.warn( + f"`{old_name}` is deprecated and will be removed in a future version; use `{new_name}` instead.", + DeprecationWarning, + stacklevel=3, + ) + return old_value + if new_supplied: + return new_value + return None + + +def _coalesce_output_from_kwarg( + output_from: Any, + output_executors: Any, +) -> Any: + """Resolve output-selection aliases to canonical ``output_from``.""" + supplied = [ + name + for name, value in ( + ("output_from", output_from), + ("output_executors", output_executors), + ) + if value is not _MISSING + ] + if len(supplied) > 1: + formatted = ", ".join(f"`{name}`" for name in supplied) + raise TypeError(f"Cannot pass multiple workflow output selection parameters ({formatted}); use `output_from`.") + + if output_executors is not _MISSING: + warnings.warn( + "`output_executors` is deprecated and will be removed in a future version; use `output_from` instead.", + DeprecationWarning, + stacklevel=3, + ) + return output_executors + if output_from is not _MISSING: + return output_from + return None + + class WorkflowRunResult(list[WorkflowEvent]): """Container for events generated during non-streaming workflow execution. @@ -73,6 +130,14 @@ class WorkflowRunResult(list[WorkflowEvent]): """ return [event.data for event in self if event.type == "output"] + def get_intermediate_outputs(self) -> list[Any]: + """Get all intermediate outputs from the workflow run result. + + Returns: + A list of intermediate outputs produced by the workflow during its execution. + """ + return [event.data for event in self if event.type == "intermediate"] + def get_request_info_events(self) -> list[WorkflowEvent[Any]]: """Get all request info events from the workflow run result. @@ -102,6 +167,42 @@ class WorkflowRunResult(list[WorkflowEvent]): # region Workflow +@dataclass(frozen=True) +class OutputDesignation: + """Immutable rule for labeling executor yields as terminal, intermediate, or hidden outputs. + + ``outputs`` is ``None`` in omitted-selection compatibility mode (every yield is terminal). In explicit mode, + ``outputs`` and ``intermediates`` are disjoint executor ID sets; unlisted executor + yields are hidden from caller-facing output/intermediate events. + Package-internal value type owned by ``Workflow``; not exported from ``agent_framework``. + """ + + outputs: frozenset[str] | None = field(default=None) + intermediates: frozenset[str] = field(default_factory=lambda: frozenset[str]()) + + def is_terminal(self, executor_id: str) -> bool: + """Return True when ``executor_id``'s yields should be labeled type='output'.""" + if self.outputs is None: + return True + return executor_id in self.outputs + + def is_intermediate(self, executor_id: str) -> bool: + """Return True when ``executor_id``'s yields should be labeled type='intermediate'.""" + if self.outputs is None: + return False + return executor_id in self.intermediates + + def classify(self, executor_id: str) -> Literal["output", "intermediate"] | None: + """Return the workflow event type for this executor's yield, or None when hidden.""" + if self.outputs is None: + return "output" + if executor_id in self.outputs: + return "output" + if executor_id in self.intermediates: + return "intermediate" + return None + + class Workflow(DictConvertible): """A graph-based execution engine that orchestrates connected executors. @@ -182,7 +283,11 @@ class Workflow(DictConvertible): name: str, description: str | None = None, max_iterations: int = DEFAULT_MAX_ITERATIONS, - output_executors: list[str] | None = None, + output_from: list[str] | None = _MISSING, + intermediate_output_from: list[str] | None = _MISSING, + *, + output_executors: list[str] | None = _MISSING, + intermediate_executors: list[str] | None = _MISSING, ): """Initialize the workflow with a list of edges. @@ -198,9 +303,21 @@ class Workflow(DictConvertible): better observability and management. description: Optional description of what the workflow does. If the workflow is built using WorkflowBuilder, this will be the description of the builder. - output_executors: Optional list of executor IDs whose outputs will be considered workflow outputs. - If None or empty, all executor outputs are treated as workflow outputs. + output_from: List of executor IDs designated as workflow outputs, or + ``None`` for omitted-selection compatibility behavior when ``intermediate_output_from`` is also + ``None``. + intermediate_output_from: List of executor IDs designated as intermediate outputs. + In explicit designation mode, unlisted executor yields are hidden from + caller-facing output/intermediate events. + output_executors: Deprecated alias for ``output_from``. Will be removed + in a future version. + intermediate_executors: Deprecated alias for ``intermediate_output_from``. Will be + removed in a future version. """ + output_from = _coalesce_output_from_kwarg(output_from, output_executors) + intermediate_output_from = _coalesce_renamed_kwarg( + "intermediate_executors", intermediate_executors, "intermediate_output_from", intermediate_output_from + ) self.edge_groups = list(edge_groups) self.executors = dict(executors) self.start_executor_id = start_executor.id @@ -215,12 +332,20 @@ class Workflow(DictConvertible): self.graph_signature = self._compute_graph_signature() self.graph_signature_hash = self._hash_graph_signature(self.graph_signature) - # Output events (WorkflowEvent with type='output') from these executors are treated as workflow outputs. - # If None or empty, all executor outputs are considered workflow outputs. - self._output_executors = list(output_executors) if output_executors else list(self.executors.keys()) + # Single value type encodes omitted-selection compatibility vs explicit output-designation policy. + output_designation_ids = ( + frozenset(output_from) + if output_from is not None + else (frozenset[str]() if intermediate_output_from is not None else None) + ) + self._output_designation: OutputDesignation = OutputDesignation( + outputs=output_designation_ids, + intermediates=frozenset(intermediate_output_from or []), + ) # Store non-serializable runtime objects as private attributes self._runner_context = runner_context + self._runner_context.set_yield_output_classifier(self._output_designation.classify) self._state = State() self._runner: Runner = Runner( self.edge_groups, @@ -254,7 +379,12 @@ class Workflow(DictConvertible): "max_iterations": self.max_iterations, "edge_groups": [group.to_dict() for group in self.edge_groups], "executors": {executor_id: executor.to_dict() for executor_id, executor in self.executors.items()}, - "output_executors": self._output_executors, + "output_executors": ( + sorted(self._output_designation.outputs) if self._output_designation.outputs is not None else None + ), + "intermediate_executors": ( + sorted(self._output_designation.intermediates) if self._output_designation.outputs is not None else None + ), } if self.description is not None: @@ -289,8 +419,44 @@ class Workflow(DictConvertible): return self.executors[self.start_executor_id] def get_output_executors(self) -> list[Executor]: - """Get the list of output executors in the workflow.""" - return [self.executors[executor_id] for executor_id in self._output_executors] + """Get the list of output executors in the workflow. + + In omitted-selection compatibility mode (no explicit ``output_from``), returns every + executor in the workflow. In explicit mode, returns only the designated output executors. + """ + designated = self._output_designation.outputs + if designated is None: + return list(self.executors.values()) + return [self._get_designated_executor(executor_id, kind="Output") for executor_id in designated] + + def get_intermediate_executors(self) -> list[Executor]: + """Get the list of intermediate executors in the workflow.""" + return [ + self._get_designated_executor(executor_id, kind="Intermediate") + for executor_id in self._output_designation.intermediates + ] + + def _get_designated_executor(self, executor_id: str, *, kind: str) -> Executor: + try: + return self.executors[executor_id] + except KeyError as exc: + raise WorkflowValidationError( + f"{kind} executor '{executor_id}' is not present in the workflow graph", + validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, + ) from exc + + def is_terminal_executor(self, executor_id: str) -> bool: + """Return True when ``executor_id``'s yields are labeled type='output'. + + Public read-only predicate over the workflow's output designation. External + observers (e.g., orchestration tests, DevUI mappers) should consult this rather + than re-encoding the rule as a set-membership check. + """ + return self._output_designation.is_terminal(executor_id) + + def is_intermediate_executor(self, executor_id: str) -> bool: + """Return True when ``executor_id``'s yields are labeled type='intermediate'.""" + return self._output_designation.is_intermediate(executor_id) def get_executors_list(self) -> list[Executor]: """Get the list of executors in the workflow.""" @@ -631,8 +797,6 @@ class Workflow(DictConvertible): function_invocation_kwargs=function_invocation_kwargs, client_kwargs=client_kwargs, ): - if event.type == "output" and not self._should_yield_output_event(event): - continue if event.type == "request_info" and event.request_id in (responses or {}): # Don't yield request_info events for which we have responses to send - # these are considered "handled". This prevents the caller from seeing @@ -825,22 +989,6 @@ class Workflow(DictConvertible): ) return {GLOBAL_KWARGS_KEY: dict(kwargs)} - def _should_yield_output_event(self, event: WorkflowEvent[Any]) -> bool: - """Determine if an output event should be yielded as a workflow output. - - Args: - event: The WorkflowEvent with type='output' to evaluate. - - Returns: - True if the event should be yielded as a workflow output, False otherwise. - """ - # If no specific output executors are defined, yield all outputs - if not self._output_executors: - return True - - # Check if the event's source executor is in the list of output executors - return event.executor_id in self._output_executors - # Graph signature helpers def _compute_graph_signature(self) -> dict[str, Any]: diff --git a/python/packages/core/agent_framework/_workflows/_workflow_builder.py b/python/packages/core/agent_framework/_workflows/_workflow_builder.py index 1a71b3a49b..942f3010f0 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow_builder.py +++ b/python/packages/core/agent_framework/_workflows/_workflow_builder.py @@ -3,8 +3,9 @@ import logging import sys import uuid +import warnings from collections.abc import Callable, Sequence -from typing import Any +from typing import Any, Literal from .._agents import SupportsAgentRun from ..observability import OtelAttr, capture_exception, create_workflow_span @@ -27,8 +28,12 @@ from ._edge import ( ) from ._executor import Executor from ._runner_context import InProcRunnerContext -from ._validation import validate_workflow_graph -from ._workflow import Workflow +from ._validation import ValidationTypeEnum, WorkflowValidationError, validate_workflow_graph +from ._workflow import ( + _MISSING, # pyright: ignore[reportPrivateUsage] + Workflow, + _coalesce_output_from_kwarg, # pyright: ignore[reportPrivateUsage] +) if sys.version_info >= (3, 11): from typing import Self # type: ignore # pragma: no cover @@ -38,6 +43,12 @@ else: logger = logging.getLogger(__name__) +_ALL_OUTPUTS: Literal["all"] = "all" +_ALL_OTHER_OUTPUTS: Literal["all_other"] = "all_other" +_OutputSelection = list[Executor | SupportsAgentRun] | Literal["all"] | None +_IntermediateOutputSelection = list[Executor | SupportsAgentRun] | Literal["all", "all_other"] | None +_AnyOutputSelection = _OutputSelection | _IntermediateOutputSelection + class WorkflowBuilder: """A builder class for constructing workflows. @@ -83,7 +94,9 @@ class WorkflowBuilder: *, start_executor: Executor | SupportsAgentRun, checkpoint_storage: CheckpointStorage | None = None, - output_executors: list[Executor | SupportsAgentRun] | None = None, + output_from: list[Executor | SupportsAgentRun] | Literal["all"] | None = _MISSING, + intermediate_output_from: _IntermediateOutputSelection = _MISSING, + output_executors: list[Executor | SupportsAgentRun] | None = _MISSING, ): """Initialize the WorkflowBuilder. @@ -98,9 +111,39 @@ class WorkflowBuilder: start_executor: The starting executor for the workflow. Can be an Executor instance or SupportsAgentRun instance. checkpoint_storage: Optional checkpoint storage for enabling workflow state persistence. - output_executors: Optional list of executors whose outputs should be collected. - If not provided, outputs from all executors are collected. + output_from: Designates which executors emit workflow output + (``type='output'`` workflow events). Pass ``"all"`` to explicitly select every + executor with declared workflow output types. + intermediate_output_from: Designates which executors emit intermediate output + (``type='intermediate'`` workflow events). Pass ``"all"`` to select every executor + with declared workflow output types as intermediate (no executor emits ``output``). + Pass ``"all_other"`` to select every executor with declared workflow output types + that is not selected by ``output_from``. + If neither ``output_from`` nor ``intermediate_output_from`` is provided, + omitted-selection compatibility behavior applies and every ``yield_output`` produces + ``type='output'``. If either is provided, explicit mode applies: listed + workflow-output executors emit ``output``, listed intermediate executors emit + ``intermediate``, and unlisted executor yields are hidden. + + Output selection behavior: + - Omit both selections: every ``yield_output`` emits ``output`` for compatibility, + with a deprecation warning. + - ``output_from="all"``: every output-capable executor emits ``output``. + - ``output_from=[A]``: only A emits ``output``; other executor payloads are hidden. + - ``output_from=[A], intermediate_output_from="all_other"``: A emits ``output``; + all other output-capable executors emit ``intermediate``. + - ``intermediate_output_from="all_other"``: no executor emits ``output``; every + output-capable executor emits ``intermediate``. + - ``output_from=[], intermediate_output_from="all_other"``: no executor emits + ``output``; every output-capable executor emits ``intermediate``. + - ``output_from=[A], intermediate_output_from=[B, C]``: A emits ``output``; B and C + emit ``intermediate``; other executor payloads are hidden. + output_executors: **Deprecated** alias for ``output_from``. Will be removed in a + future version. """ + output_from = _coalesce_output_from_kwarg(output_from, output_executors) + if intermediate_output_from is _MISSING: + intermediate_output_from = None self._edge_groups: list[EdgeGroup] = [] self._executors: dict[str, Executor] = {} self._start_executor: Executor | None = None @@ -113,8 +156,13 @@ class WorkflowBuilder: # being created for the same agent. self._agent_wrappers: dict[str, Executor] = {} - # Output executors filter; if set, only outputs from these executors are yielded - self._output_executors: list[Executor | SupportsAgentRun] = output_executors if output_executors else [] + # ``None`` for both means omitted-selection compatibility behavior + # (every yield_output produces type='output'). + # If either is provided, explicit mode applies and unlisted executor yields are hidden. + self._output_from: _OutputSelection = self._coerce_output_from(output_from) + self._intermediate_output_from: _IntermediateOutputSelection = self._coerce_intermediate_output_from( + intermediate_output_from + ) # Set the start executor self._set_start_executor(start_executor) @@ -584,6 +632,96 @@ class WorkflowBuilder: if existing is not wrapped: self._add_executor(wrapped) + def _coerce_output_from(self, output_from: Any) -> _OutputSelection: + """Coerce workflow-output selection while preserving the explicit ``"all"`` literal.""" + if output_from is None: + return None + if output_from == _ALL_OUTPUTS: + return _ALL_OUTPUTS + if isinstance(output_from, str): + raise ValueError(f"Unsupported output_from literal {output_from!r}; use 'all' or a list of executors.") + return list(output_from) + + def _coerce_intermediate_output_from(self, intermediate_output_from: Any) -> _IntermediateOutputSelection: + """Coerce intermediate-output selection and reject output-only literals.""" + if intermediate_output_from is None: + return None + if isinstance(intermediate_output_from, str): + if intermediate_output_from == _ALL_OUTPUTS: + return _ALL_OUTPUTS + if intermediate_output_from == _ALL_OTHER_OUTPUTS: + return _ALL_OTHER_OUTPUTS + raise ValueError( + f"Unsupported intermediate_output_from literal {intermediate_output_from!r}; " + "use 'all', 'all_other', or a list of executors." + ) + return list(intermediate_output_from) + + def _resolve_designated_executor_ids( + self, + designated: _AnyOutputSelection, + ) -> list[str] | None: + """Resolve an optional designation list into executor IDs without mutating the graph.""" + if designated is None: + return None + if designated == _ALL_OUTPUTS: + return [executor_id for executor_id, executor in self._executors.items() if executor.workflow_output_types] + if designated == _ALL_OTHER_OUTPUTS: + raise ValueError("intermediate_output_from='all_other' must be expanded relative to output_from.") + ids: list[str] = [] + for item in designated: + if isinstance(item, Executor): + ids.append(item.id) + elif isinstance(item, SupportsAgentRun): + ids.append(resolve_agent_id(item)) + else: + raise TypeError( + "WorkflowBuilder expected designation entries to be Executor or SupportsAgentRun instances; " + f"got {type(item).__name__}." + ) + return ids + + def _validate_designation_lists( + self, + output_executor_ids: list[str] | None, + intermediate_executor_ids: list[str] | None, + ) -> None: + """Validate builder-level designation rules that need omitted-vs-explicit context.""" + explicit_mode = output_executor_ids is not None or intermediate_executor_ids is not None + if not explicit_mode: + return + + output_ids = output_executor_ids or [] + intermediate_ids = intermediate_executor_ids or [] + if not output_ids and not intermediate_ids: + raise WorkflowValidationError( + "Explicit workflow output designation must include at least one output or intermediate executor.", + validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, + ) + + duplicate_outputs = sorted({executor_id for executor_id in output_ids if output_ids.count(executor_id) > 1}) + if duplicate_outputs: + raise WorkflowValidationError( + f"Duplicate output executor designation(s): {duplicate_outputs}", + validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, + ) + + duplicate_intermediates = sorted({ + executor_id for executor_id in intermediate_ids if intermediate_ids.count(executor_id) > 1 + }) + if duplicate_intermediates: + raise WorkflowValidationError( + f"Duplicate intermediate executor designation(s): {duplicate_intermediates}", + validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, + ) + + overlap = sorted(set(output_ids).intersection(intermediate_ids)) + if overlap: + raise WorkflowValidationError( + f"Executors cannot be both output and intermediate designated: {overlap}", + validation_type=ValidationTypeEnum.OUTPUT_VALIDATION, + ) + def build(self) -> Workflow: """Build and return the constructed workflow. @@ -625,6 +763,43 @@ class WorkflowBuilder: # Workflows can be reused multiple times events2 = await workflow.run("world") print(events2.get_outputs()) # ['WORLD'] + + # Select one executor as Workflow Output. + workflow = WorkflowBuilder(start_executor=executor, output_from=[executor]).build() + events = await workflow.run("hello") + print(events.get_outputs()) # ['HELLO'] + print(events.get_intermediate_outputs()) # [] + + # Make one executor Workflow Output and every other output-capable executor Intermediate Output. + workflow = ( + WorkflowBuilder( + start_executor=planner, + output_from=[answerer], + intermediate_output_from="all_other", + ) + .add_edge(planner, answerer) + .build() + ) + events = await workflow.run("hello") + print(events.get_outputs()) # outputs from answerer + print(events.get_intermediate_outputs()) # outputs from planner + + # Build a progress-only workflow: no Workflow Output, all output-capable executors are intermediate. + workflow = ( + WorkflowBuilder(start_executor=planner, intermediate_output_from="all_other") + .add_edge(planner, answerer) + .build() + ) + events = await workflow.run("hello") + print(events.get_outputs()) # [] + print(events.get_intermediate_outputs()) # outputs from planner and answerer + + # Explicitly preserve all-output behavior without relying on omitted-selection compatibility. + workflow = ( + WorkflowBuilder(start_executor=planner, output_from="all").add_edge(planner, answerer).build() + ) + events = await workflow.run("hello") + print(events.get_outputs()) # outputs from planner and answerer """ # Create workflow build span that includes validation and workflow creation with create_workflow_span(OtelAttr.WORKFLOW_BUILD_SPAN) as span: @@ -637,19 +812,47 @@ class WorkflowBuilder: "Starting executor must be set via the start_executor constructor parameter before building." ) + if self._output_from is None and self._intermediate_output_from is None: + warnings.warn( + "WorkflowBuilder built without explicit output_from or intermediate_output_from; " + "every yield_output produces type='output' for compatibility. Pass output_from='all', " + "output_from=[...], or intermediate_output_from=[...] to opt into explicit designation - " + "explicit designation will be required in a future version.", + DeprecationWarning, + stacklevel=2, + ) + start_executor = self._start_executor executors = self._executors edge_groups = self._edge_groups - output_executors = [ex.id for ex in self._output_executors if isinstance(ex, Executor)] + [ - resolve_agent_id(agent) for agent in self._output_executors if isinstance(agent, SupportsAgentRun) - ] + output_ids = self._resolve_designated_executor_ids(self._output_from) + intermediate_output_ids: list[str] | None + if self._intermediate_output_from == _ALL_OTHER_OUTPUTS: + output_ids_for_all_other = output_ids or [] + intermediate_output_ids = [ + executor_id + for executor_id, executor in self._executors.items() + if executor.workflow_output_types and executor_id not in output_ids_for_all_other + ] + else: + intermediate_output_ids = self._resolve_designated_executor_ids(self._intermediate_output_from) + self._validate_designation_lists(output_ids, intermediate_output_ids) + + explicit_mode = output_ids is not None or intermediate_output_ids is not None + output_for_workflow: list[str] | None = output_ids if explicit_mode else None + if explicit_mode and output_for_workflow is None: + output_for_workflow = [] + intermediate_output_for_workflow: list[str] | None = intermediate_output_ids if explicit_mode else None + if explicit_mode and intermediate_output_for_workflow is None: + intermediate_output_for_workflow = [] # Perform validation before creating the workflow validate_workflow_graph( edge_groups, executors, start_executor, - output_executors, + output_for_workflow or [], + intermediate_output_for_workflow or [], ) # Add validation completed event @@ -666,7 +869,8 @@ class WorkflowBuilder: self._name, description=self._description, max_iterations=self._max_iterations, - output_executors=output_executors, + output_from=output_for_workflow, + intermediate_output_from=intermediate_output_for_workflow, ) build_attributes: dict[str, Any] = { OtelAttr.WORKFLOW_BUILDER_NAME: self._name, diff --git a/python/packages/core/agent_framework/_workflows/_workflow_context.py b/python/packages/core/agent_framework/_workflows/_workflow_context.py index 51add07a5c..bfc8601e5d 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow_context.py +++ b/python/packages/core/agent_framework/_workflows/_workflow_context.py @@ -201,6 +201,7 @@ def validate_workflow_context_annotation( # Event types reserved for framework lifecycle (not allowed from user code) _FRAMEWORK_LIFECYCLE_EVENT_TYPES: frozenset[str] = frozenset({"started", "status", "failed"}) +_OUTPUT_SELECTION_EVENT_TYPES: frozenset[str] = frozenset({"output", "intermediate"}) class WorkflowContext(Generic[OutT, W_OutT]): @@ -337,7 +338,20 @@ class WorkflowContext(Generic[OutT, W_OutT]): await self._runner_context.send_message(msg) async def yield_output(self, output: W_OutT) -> None: - """Set the output of the workflow. + """Yield an output from this executor. + + The framework labels the resulting workflow event based on the workflow's explicit + output designation: + + - Omitted-selection compatibility behavior: every yield produces ``type='output'``. + - Explicit mode: output-designated executors produce ``type='output'``, + intermediate-designated executors produce ``type='intermediate'``, and + unlisted executor yields are hidden from caller-facing events. + + Whether a given executor produces ``output`` or ``intermediate`` events is fixed at + workflow-build time via ``output_from`` / ``intermediate_output_from`` on + :class:`WorkflowBuilder`; an executor cannot vary the label per yield. To change an + executor's role, list it under a different designation when building the workflow. Args: output: The output to yield. This must conform to the workflow output type(s) @@ -347,12 +361,24 @@ class WorkflowContext(Generic[OutT, W_OutT]): # (deepcopy to capture state at yield time) self._yielded_outputs.append(copy.deepcopy(output)) + event_type = self._runner_context.classify_yielded_output(self._executor_id) + if event_type is None: + return + with _framework_event_origin(): - event = WorkflowEvent.output(self._executor_id, output) + event = WorkflowEvent(event_type, executor_id=self._executor_id, data=output) await self._runner_context.add_event(event) async def add_event(self, event: WorkflowEvent[Any]) -> None: """Add an event to the workflow context.""" + if event.origin == WorkflowEventSource.EXECUTOR and event.type in _OUTPUT_SELECTION_EVENT_TYPES: + warning_msg = ( + f"Executor '{self._executor_id}' attempted to emit a '{event.type}' event directly, " + "which is reserved for ctx.yield_output(). The event was ignored." + ) + logger.warning(warning_msg) + await self._runner_context.add_event(WorkflowEvent.warning(warning_msg)) + return if event.origin == WorkflowEventSource.EXECUTOR and event.type in _FRAMEWORK_LIFECYCLE_EVENT_TYPES: warning_msg = ( f"Executor '{self._executor_id}' attempted to emit a '{event.type}' event, " diff --git a/python/packages/core/agent_framework/_workflows/_workflow_executor.py b/python/packages/core/agent_framework/_workflows/_workflow_executor.py index 847b44863c..e131533429 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow_executor.py +++ b/python/packages/core/agent_framework/_workflows/_workflow_executor.py @@ -16,6 +16,7 @@ from ._const import GLOBAL_KWARGS_KEY, WORKFLOW_RUN_KWARGS_KEY from ._events import ( WorkflowEvent, WorkflowRunState, + _framework_event_origin, # type: ignore[reportPrivateUsage] ) from ._executor import Executor, handler from ._request_info_mixin import response_handler @@ -552,10 +553,12 @@ class WorkflowExecutor(Executor): # Collect all events from the workflow request_info_events = result.get_request_info_events() outputs = result.get_outputs() + intermediate_outputs = result.get_intermediate_outputs() workflow_run_state = result.get_final_state() logger.debug( f"WorkflowExecutor {self.id} processing workflow result with " - f"{len(outputs)} outputs and {len(request_info_events)} request info events. " + f"{len(outputs)} outputs, {len(intermediate_outputs)} intermediate outputs, " + f"and {len(request_info_events)} request info events. " f"Workflow run state: {workflow_run_state}" ) @@ -566,6 +569,19 @@ class WorkflowExecutor(Executor): else: await asyncio.gather(*[ctx.send_message(output) for output in outputs]) + # Pipe sub-workflow intermediate emissions up through the parent's event stream. + # Bypasses the parent's yield-output classifier so the 'intermediate' label is preserved + # across the encapsulation boundary; uses this WorkflowExecutor's id as the source + # so outer callers don't need to know the sub-workflow's internal executor layout. + if intermediate_outputs: + + async def _forward_intermediate_output(output: Any) -> None: + with _framework_event_origin(): + event = WorkflowEvent("intermediate", executor_id=self.id, data=output) + await ctx.add_event(event) + + await asyncio.gather(*[_forward_intermediate_output(output) for output in intermediate_outputs]) + # Process request info events for event in request_info_events: request_id = event.request_id diff --git a/python/packages/core/tests/workflow/test_agent_executor_tool_calls.py b/python/packages/core/tests/workflow/test_agent_executor_tool_calls.py index 07a37f9617..9f17af9e4e 100644 --- a/python/packages/core/tests/workflow/test_agent_executor_tool_calls.py +++ b/python/packages/core/tests/workflow/test_agent_executor_tool_calls.py @@ -272,9 +272,7 @@ async def test_agent_executor_tool_call_with_approval() -> None: tools=[mock_tool_requiring_approval], ) - workflow = ( - WorkflowBuilder(start_executor=agent, output_executors=[test_executor]).add_edge(agent, test_executor).build() - ) + workflow = WorkflowBuilder(start_executor=agent, output_from=[test_executor]).add_edge(agent, test_executor).build() # Act events = await workflow.run("Invoke tool requiring approval") @@ -343,9 +341,7 @@ async def test_agent_executor_parallel_tool_call_with_approval() -> None: tools=[mock_tool_requiring_approval], ) - workflow = ( - WorkflowBuilder(start_executor=agent, output_executors=[test_executor]).add_edge(agent, test_executor).build() - ) + workflow = WorkflowBuilder(start_executor=agent, output_from=[test_executor]).add_edge(agent, test_executor).build() # Act events = await workflow.run("Invoke tool requiring approval") @@ -512,9 +508,7 @@ async def test_agent_executor_declaration_only_tool_emits_request_info() -> None tools=[declaration_only_tool], ) - workflow = ( - WorkflowBuilder(start_executor=agent, output_executors=[test_executor]).add_edge(agent, test_executor).build() - ) + workflow = WorkflowBuilder(start_executor=agent, output_from=[test_executor]).add_edge(agent, test_executor).build() # Act events = await workflow.run("Use the client side tool") @@ -587,9 +581,7 @@ async def test_agent_executor_parallel_declaration_only_tool_emits_request_info( tools=[declaration_only_tool], ) - workflow = ( - WorkflowBuilder(start_executor=agent, output_executors=[test_executor]).add_edge(agent, test_executor).build() - ) + workflow = WorkflowBuilder(start_executor=agent, output_from=[test_executor]).add_edge(agent, test_executor).build() # Act events = await workflow.run("Use the client side tool") diff --git a/python/packages/core/tests/workflow/test_agent_run_event_typing.py b/python/packages/core/tests/workflow/test_agent_run_event_typing.py index 2b16f01258..b40e5d91ba 100644 --- a/python/packages/core/tests/workflow/test_agent_run_event_typing.py +++ b/python/packages/core/tests/workflow/test_agent_run_event_typing.py @@ -9,7 +9,7 @@ from agent_framework._workflows._events import WorkflowEvent def test_workflow_event_with_agent_response_data_type() -> None: """Verify WorkflowEvent[AgentResponse].data is typed as AgentResponse.""" response = AgentResponse(messages=[Message(role="assistant", contents=["Hello"])]) - event: WorkflowEvent[AgentResponse] = WorkflowEvent.emit(executor_id="test", data=response) + event: WorkflowEvent[AgentResponse] = WorkflowEvent("intermediate", executor_id="test", data=response) # This assignment should pass type checking without a cast data: AgentResponse = event.data @@ -20,7 +20,7 @@ def test_workflow_event_with_agent_response_data_type() -> None: def test_workflow_event_with_agent_response_update_data_type() -> None: """Verify WorkflowEvent[AgentResponseUpdate].data is typed as AgentResponseUpdate.""" update = AgentResponseUpdate() - event: WorkflowEvent[AgentResponseUpdate] = WorkflowEvent.emit(executor_id="test", data=update) + event: WorkflowEvent[AgentResponseUpdate] = WorkflowEvent("intermediate", executor_id="test", data=update) # This assignment should pass type checking without a cast data: AgentResponseUpdate = event.data @@ -30,7 +30,7 @@ def test_workflow_event_with_agent_response_update_data_type() -> None: def test_workflow_event_repr() -> None: """Verify WorkflowEvent.__repr__ uses consistent format.""" response = AgentResponse(messages=[Message(role="assistant", contents=["Hello"])]) - event: WorkflowEvent[AgentResponse] = WorkflowEvent.emit(executor_id="test", data=response) + event: WorkflowEvent[AgentResponse] = WorkflowEvent("intermediate", executor_id="test", data=response) repr_str = repr(event) assert "WorkflowEvent" in repr_str diff --git a/python/packages/core/tests/workflow/test_full_conversation.py b/python/packages/core/tests/workflow/test_full_conversation.py index 79d8626bc2..27b3dc4019 100644 --- a/python/packages/core/tests/workflow/test_full_conversation.py +++ b/python/packages/core/tests/workflow/test_full_conversation.py @@ -177,7 +177,7 @@ async def test_agent_executor_populates_full_conversation_non_streaming() -> Non agent_exec = AgentExecutor(agent, id="agent1-exec") capturer = _CaptureFullConversation(id="capture") - wf = WorkflowBuilder(start_executor=agent_exec, output_executors=[capturer]).add_edge(agent_exec, capturer).build() + wf = WorkflowBuilder(start_executor=agent_exec, output_from=[capturer]).add_edge(agent_exec, capturer).build() # Act: use run() to test non-streaming mode result = await wf.run("hello world") @@ -344,7 +344,7 @@ async def test_agent_executor_full_conversation_round_trip_does_not_duplicate_hi coordinator = _RoundTripCoordinator(target_agent_id="writer_agent") wf = ( - WorkflowBuilder(start_executor=agent_exec, output_executors=[coordinator]) + WorkflowBuilder(start_executor=agent_exec, output_from=[coordinator]) .add_edge(agent_exec, coordinator) .add_edge(coordinator, agent_exec) .build() @@ -450,7 +450,7 @@ async def test_run_request_with_full_history_clears_service_session_id() -> None coordinator = _FullHistoryReplayCoordinator(id="coord", target_exec=spy_exec) wf = ( - WorkflowBuilder(start_executor=tool_exec, output_executors=[coordinator]) + WorkflowBuilder(start_executor=tool_exec, output_from=[coordinator]) .add_edge(tool_exec, coordinator) .add_edge(coordinator, spy_exec) .build() @@ -478,7 +478,7 @@ async def test_from_response_preserves_service_session_id() -> None: # Simulate a prior run on the spy executor. spy_exec._session.service_session_id = "resp_PREVIOUS_RUN" # pyright: ignore[reportPrivateUsage] - wf = WorkflowBuilder(start_executor=tool_exec, output_executors=[spy_exec]).add_edge(tool_exec, spy_exec).build() + wf = WorkflowBuilder(start_executor=tool_exec, output_from=[spy_exec]).add_edge(tool_exec, spy_exec).build() result = await wf.run("start") assert result.get_outputs() is not None @@ -517,7 +517,7 @@ async def test_with_text_preserves_full_conversation_through_custom_executor() - capturer = _CaptureFullConversation(id="capture") wf = ( - WorkflowBuilder(start_executor=agent1, output_executors=[capturer]) + WorkflowBuilder(start_executor=agent1, output_from=[capturer]) .add_chain([agent1, agent2, _upper_case_executor, agent3, capturer]) .build() ) diff --git a/python/packages/core/tests/workflow/test_functional_workflow.py b/python/packages/core/tests/workflow/test_functional_workflow.py index ba465ffe0b..6502a0e353 100644 --- a/python/packages/core/tests/workflow/test_functional_workflow.py +++ b/python/packages/core/tests/workflow/test_functional_workflow.py @@ -165,13 +165,13 @@ class TestEventEmission: @workflow async def pipeline(x: int, ctx: RunContext) -> int: - await ctx.add_event(WorkflowEvent.emit("pipeline", "custom_data")) + await ctx.add_event(WorkflowEvent("intermediate", executor_id="pipeline", data="custom_data")) return x result = await pipeline.run(1) - data_events = [e for e in result if e.type == "data"] - assert len(data_events) == 1 - assert data_events[0].data == "custom_data" + intermediate_events = [e for e in result if e.type == "intermediate"] + assert len(intermediate_events) == 1 + assert intermediate_events[0].data == "custom_data" # --------------------------------------------------------------------------- diff --git a/python/packages/core/tests/workflow/test_output_designation.py b/python/packages/core/tests/workflow/test_output_designation.py new file mode 100644 index 0000000000..cd4b23ee21 --- /dev/null +++ b/python/packages/core/tests/workflow/test_output_designation.py @@ -0,0 +1,137 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for the ``OutputDesignation`` value type and the ``Workflow.is_terminal_executor`` +public predicate that delegates to it. + +The states the value type encodes: +- Omitted-selection compatibility: ``outputs=None`` -> every executor is terminal. +- Explicit: disjoint ``outputs`` and ``intermediates`` sets classify listed executors, + and hide unlisted executors. +""" + +from __future__ import annotations + +import pytest +from typing_extensions import Never + +from agent_framework import ( + Message, + WorkflowBuilder, + WorkflowContext, + WorkflowValidationError, + executor, +) +from agent_framework._workflows._runner_context import InProcRunnerContext +from agent_framework._workflows._workflow import OutputDesignation, Workflow + +# --------------------------------------------------------------------------- +# OutputDesignation value type +# --------------------------------------------------------------------------- + + +def test_omitted_selection_designation_marks_every_executor_as_terminal() -> None: + designation = OutputDesignation() # designated defaults to None + assert designation.outputs is None + assert designation.is_terminal("anything") + assert designation.is_terminal("else") + assert designation.classify("anything") == "output" + + +def test_strict_empty_designation_marks_no_executor_as_terminal() -> None: + designation = OutputDesignation(outputs=frozenset()) + assert designation.outputs == frozenset() + assert not designation.is_terminal("anything") + assert not designation.is_terminal("else") + assert designation.classify("anything") is None + + +def test_strict_designated_set_only_terminal_for_members() -> None: + designation = OutputDesignation(outputs=frozenset({"alpha", "beta"}), intermediates=frozenset({"gamma"})) + assert designation.is_terminal("alpha") + assert designation.is_terminal("beta") + assert not designation.is_terminal("gamma") + assert designation.is_intermediate("gamma") + assert designation.classify("alpha") == "output" + assert designation.classify("gamma") == "intermediate" + assert designation.classify("delta") is None + + +def test_designation_is_frozen() -> None: + from dataclasses import FrozenInstanceError + + designation = OutputDesignation(outputs=frozenset({"alpha"})) + with pytest.raises(FrozenInstanceError): + designation.outputs = frozenset({"beta"}) # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# Workflow.is_terminal_executor delegates to the designation +# --------------------------------------------------------------------------- + + +@executor +async def _emit_one(messages: list[Message], ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("hello") + + +@executor +async def _downstream(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("downstream") + + +def test_is_terminal_executor_omitted_selection_returns_true_for_any_id() -> None: + """Omitted-selection compatibility behavior: every executor is terminal.""" + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + workflow = WorkflowBuilder(start_executor=_emit_one).build() + assert workflow.is_terminal_executor(_emit_one.id) + assert workflow.is_terminal_executor("anything-else") + + +def test_is_intermediate_executor_explicit_list_returns_true_only_for_designated() -> None: + """Explicit mode tracks intermediate-designated executors separately.""" + workflow = WorkflowBuilder(start_executor=_emit_one, intermediate_output_from=[_emit_one]).build() + assert not workflow.is_terminal_executor(_emit_one.id) + assert not workflow.is_terminal_executor("nope") + assert workflow.is_intermediate_executor(_emit_one.id) + assert not workflow.is_intermediate_executor("nope") + + +def test_is_terminal_executor_strict_list_returns_true_only_for_designated() -> None: + """Strict mode with a designated list: only listed executors are terminal.""" + workflow = ( + WorkflowBuilder(start_executor=_emit_one, output_from=[_emit_one]).add_edge(_emit_one, _downstream).build() + ) + assert workflow.is_terminal_executor(_emit_one.id) + assert not workflow.is_terminal_executor(_downstream.id) + + +def test_get_output_executors_throws_when_designation_references_missing_executor() -> None: + workflow = Workflow( + [], + {_emit_one.id: _emit_one}, + _emit_one, + InProcRunnerContext(), + "test", + output_from=["missing"], + ) + + with pytest.raises(WorkflowValidationError, match="Output executor 'missing' is not present"): + workflow.get_output_executors() + + +def test_get_intermediate_executors_throws_when_designation_references_missing_executor() -> None: + workflow = Workflow( + [], + {_emit_one.id: _emit_one}, + _emit_one, + InProcRunnerContext(), + "test", + output_from=[], + intermediate_output_from=["missing"], + ) + + with pytest.raises(WorkflowValidationError, match="Intermediate executor 'missing' is not present"): + workflow.get_intermediate_executors() diff --git a/python/packages/core/tests/workflow/test_output_executors_contract.py b/python/packages/core/tests/workflow/test_output_executors_contract.py new file mode 100644 index 0000000000..31f4f946b7 --- /dev/null +++ b/python/packages/core/tests/workflow/test_output_executors_contract.py @@ -0,0 +1,287 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for the explicit output/intermediate selection contract on WorkflowBuilder.""" + +from __future__ import annotations + +import warnings +from typing import Any + +import pytest +from typing_extensions import Never + +from agent_framework import ( + Message, + WorkflowBuilder, + WorkflowContext, + WorkflowValidationError, + executor, +) + + +@executor +async def _emit_one(messages: list[Message], ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("hello") + + +@executor +async def _start(messages: list[Message], ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output("from-start") + await ctx.send_message("downstream") + + +@executor +async def _downstream(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("from-downstream") + + +def test_designation_unset_emits_deprecation_warning() -> None: + """State A: WorkflowBuilder built without explicit designation warns.""" + with pytest.warns(DeprecationWarning, match="output_from or intermediate_output_from") as warning_info: + WorkflowBuilder(start_executor=_emit_one).build() + assert str(warning_info[0].message) == ( + "WorkflowBuilder built without explicit output_from or intermediate_output_from; " + "every yield_output produces type='output' for compatibility. Pass output_from='all', " + "output_from=[...], or intermediate_output_from=[...] to opt into explicit designation - " + "explicit designation will be required in a future version." + ) + + +@pytest.mark.asyncio +async def test_designation_unset_preserves_compatibility_all_output_behavior() -> None: + """Omitted designation keeps compatibility all-output behavior while warning.""" + with pytest.warns(DeprecationWarning, match="output_from or intermediate_output_from"): + workflow = WorkflowBuilder(start_executor=_start).add_edge(_start, _downstream).build() + + result = await workflow.run([Message(role="user", contents=["hi"])]) + + assert result.get_outputs() == ["from-start", "from-downstream"] + assert result.get_intermediate_outputs() == [] + + +@pytest.mark.asyncio +async def test_output_from_all_emits_all_outputs_without_omitted_selection_warning() -> None: + """Explicit all-output designation emits every executor payload without omitted-selection warning.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + workflow = WorkflowBuilder(start_executor=_start, output_from="all").add_edge(_start, _downstream).build() + + result = await workflow.run([Message(role="user", contents=["hi"])]) + + assert result.get_outputs() == ["from-start", "from-downstream"] + assert result.get_intermediate_outputs() == [] + + +@pytest.mark.asyncio +async def test_output_from_all_with_empty_intermediate_list_is_valid() -> None: + """Explicit all-output plus an empty intermediate list is a concrete no-intermediate selection.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + workflow = ( + WorkflowBuilder(start_executor=_start, output_from="all", intermediate_output_from=[]) + .add_edge(_start, _downstream) + .build() + ) + + result = await workflow.run([Message(role="user", contents=["hi"])]) + + assert result.get_outputs() == ["from-start", "from-downstream"] + assert result.get_intermediate_outputs() == [] + + +@pytest.mark.asyncio +async def test_intermediate_output_from_all_other_marks_non_outputs_as_intermediate() -> None: + """All-other intermediate designation classifies every non-output executor yield as intermediate.""" + workflow = ( + WorkflowBuilder( + start_executor=_start, + output_from=[_downstream], + intermediate_output_from="all_other", + ) + .add_edge(_start, _downstream) + .build() + ) + + result = await workflow.run([Message(role="user", contents=["hi"])]) + + assert result.get_outputs() == ["from-downstream"] + assert result.get_intermediate_outputs() == ["from-start"] + + +@pytest.mark.asyncio +async def test_all_other_streaming_events_mark_non_outputs_as_intermediate() -> None: + """All-other emits intermediate events while streaming, not just in collected results.""" + workflow = ( + WorkflowBuilder( + start_executor=_start, + output_from=[_downstream], + intermediate_output_from="all_other", + ) + .add_edge(_start, _downstream) + .build() + ) + outputs: list[str] = [] + intermediates: list[str] = [] + + async for event in workflow.run([Message(role="user", contents=["hi"])], stream=True): + if event.type == "output": + outputs.append(event.data) + elif event.type == "intermediate": + intermediates.append(event.data) + + assert outputs == ["from-downstream"] + assert intermediates == ["from-start"] + + +def test_all_other_expands_to_concrete_intermediate_executor_selection_at_build_time() -> None: + """The runner receives concrete executor IDs after all-other expansion.""" + workflow = ( + WorkflowBuilder( + start_executor=_start, + output_from=[_downstream], + intermediate_output_from="all_other", + ) + .add_edge(_start, _downstream) + .build() + ) + + assert {executor.id for executor in workflow.get_output_executors()} == {_downstream.id} + assert {executor.id for executor in workflow.get_intermediate_executors()} == {_start.id} + assert workflow.is_intermediate_executor(_start.id) + assert not workflow.is_intermediate_executor(_downstream.id) + + +@pytest.mark.asyncio +async def test_all_other_with_omitted_output_from_emits_only_intermediate_outputs() -> None: + """All-other intermediate designation opts out of omitted-selection all-output behavior.""" + workflow = ( + WorkflowBuilder( + start_executor=_start, + intermediate_output_from="all_other", + ) + .add_edge(_start, _downstream) + .build() + ) + + result = await workflow.run([Message(role="user", contents=["hi"])]) + + assert result.get_outputs() == [] + assert result.get_intermediate_outputs() == ["from-start", "from-downstream"] + + +@pytest.mark.asyncio +async def test_all_other_with_empty_output_from_emits_only_intermediate_outputs() -> None: + """All-other intermediate designation treats an empty output list as selecting no workflow outputs.""" + workflow = ( + WorkflowBuilder( + start_executor=_start, + output_from=[], + intermediate_output_from="all_other", + ) + .add_edge(_start, _downstream) + .build() + ) + + result = await workflow.run([Message(role="user", contents=["hi"])]) + + assert result.get_outputs() == [] + assert result.get_intermediate_outputs() == ["from-start", "from-downstream"] + + +@pytest.mark.asyncio +async def test_all_other_with_output_from_all_expands_to_empty_intermediate_selection() -> None: + """All-other is empty when every output-capable executor is already selected as workflow output.""" + workflow = ( + WorkflowBuilder( + start_executor=_start, + output_from="all", + intermediate_output_from="all_other", + ) + .add_edge(_start, _downstream) + .build() + ) + + result = await workflow.run([Message(role="user", contents=["hi"])]) + + assert result.get_outputs() == ["from-start", "from-downstream"] + assert result.get_intermediate_outputs() == [] + + +@pytest.mark.asyncio +async def test_intermediate_output_from_all_routes_every_yield_to_intermediate() -> None: + """``intermediate_output_from="all"`` designates every output-capable executor as intermediate.""" + workflow = ( + WorkflowBuilder(start_executor=_start, intermediate_output_from="all").add_edge(_start, _downstream).build() + ) + + result = await workflow.run([Message(role="user", contents=["hi"])]) + + assert result.get_outputs() == [] + assert result.get_intermediate_outputs() == ["from-start", "from-downstream"] + + +def test_output_from_all_other_is_rejected() -> None: + """The all-other literal is only valid for intermediate output selection.""" + with pytest.raises(ValueError, match="output_from.*all_other"): + WorkflowBuilder(start_executor=_emit_one, output_from="all_other") # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("output_from", "intermediate_output_from"), + [([_emit_one], None), (None, [_emit_one]), ([], [_emit_one])], + ids=["output_list", "intermediate_list", "empty_output_with_intermediate"], +) +def test_explicit_designation_with_executor_does_not_warn(output_from, intermediate_output_from) -> None: + """State B: any explicit designation with at least one executor opts into explicit mode without warning.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + WorkflowBuilder( + start_executor=_emit_one, + output_from=output_from, + intermediate_output_from=intermediate_output_from, + ).build() + + +@pytest.mark.parametrize( + ("output_from", "intermediate_output_from"), + [([], None), (None, []), ([], [])], + ids=["empty_output", "empty_intermediate", "both_empty"], +) +def test_empty_explicit_designation_fails(output_from, intermediate_output_from) -> None: + """State C: explicit mode needs at least one output or intermediate executor.""" + with pytest.raises(WorkflowValidationError, match="at least one output or intermediate executor"): + WorkflowBuilder( + start_executor=_emit_one, + output_from=output_from, + intermediate_output_from=intermediate_output_from, + ).build() + + +def test_passing_both_output_executors_and_output_from_raises_type_error() -> None: + """State D: supplying a deprecated alias and the canonical kwarg is unambiguous user error.""" + with pytest.raises(TypeError, match="Cannot pass multiple workflow output selection parameters"): + WorkflowBuilder( + start_executor=_emit_one, + output_executors=[_emit_one], + output_from=[_emit_one], + ) + + +def test_intermediate_executors_builder_parameter_is_not_public() -> None: + """The branch-only intermediate_executors builder parameter is not supported.""" + builder_type: Any = WorkflowBuilder + with pytest.raises(TypeError, match="unexpected keyword argument 'intermediate_executors'"): + builder_type( + start_executor=_emit_one, + intermediate_executors=[_emit_one], + ) + + +def test_final_output_from_builder_parameter_is_not_public() -> None: + """The branch-only final_output_from builder parameter is not supported.""" + builder_type: Any = WorkflowBuilder + with pytest.raises(TypeError, match="unexpected keyword argument 'final_output_from'"): + builder_type( + start_executor=_emit_one, + final_output_from=[_emit_one], + ) diff --git a/python/packages/core/tests/workflow/test_runner.py b/python/packages/core/tests/workflow/test_runner.py index a42e94f39d..4fef26bd2d 100644 --- a/python/packages/core/tests/workflow/test_runner.py +++ b/python/packages/core/tests/workflow/test_runner.py @@ -158,7 +158,9 @@ async def test_runner_run_iteration_preserves_message_order_per_edge_runner() -> def __init__(self) -> None: self.received: list[int] = [] - async def send_message(self, message: WorkflowMessage, state: State, ctx: RunnerContext) -> bool: + async def send_message( + self, message: WorkflowMessage, state: State, ctx: RunnerContext, *args: object, **kwargs: object + ) -> bool: message_data = message.data assert isinstance(message_data, MockMessage) self.received.append(message_data.data) @@ -188,7 +190,9 @@ async def test_runner_run_iteration_delivers_different_edge_runners_concurrently self.release = asyncio.Event() self.call_count = 0 - async def send_message(self, message: WorkflowMessage, state: State, ctx: RunnerContext) -> bool: + async def send_message( + self, message: WorkflowMessage, state: State, ctx: RunnerContext, *args: object, **kwargs: object + ) -> bool: self.call_count += 1 self.started.set() await self.release.wait() @@ -199,7 +203,9 @@ async def test_runner_run_iteration_delivers_different_edge_runners_concurrently self.probe_completed = asyncio.Event() self.call_count = 0 - async def send_message(self, message: WorkflowMessage, state: State, ctx: RunnerContext) -> bool: + async def send_message( + self, message: WorkflowMessage, state: State, ctx: RunnerContext, *args: object, **kwargs: object + ) -> bool: self.call_count += 1 self.probe_completed.set() return True @@ -766,7 +772,7 @@ async def test_runner_with_pre_loop_events(): runner = Runner([], {}, state, ctx, "test_name", graph_signature_hash="test_hash") # Add an event before running - await ctx.add_event(WorkflowEvent.output(executor_id="test_executor", data="pre-loop-output")) + await ctx.add_event(WorkflowEvent("output", executor_id="test_executor", data="pre-loop-output")) events: list[WorkflowEvent] = [] async for event in runner.run_until_convergence(): @@ -891,7 +897,7 @@ class ExecutorThatFailsWithEvents(Executor): # First emit an output event to the workflow context await ctx.yield_output(f"output-before-failure-{message.data}") # Add some events directly to the runner context - await self._runner_ctx.add_event(WorkflowEvent.output(executor_id=self.id, data="pending-event")) + await self._runner_ctx.add_event(WorkflowEvent("output", executor_id=self.id, data="pending-event")) # Fail on the specified iteration if self._iteration_count >= self._fail_on_iteration: raise RuntimeError("Executor failed with pending events") diff --git a/python/packages/core/tests/workflow/test_serialization.py b/python/packages/core/tests/workflow/test_serialization.py index 55284db407..ed6316ecbe 100644 --- a/python/packages/core/tests/workflow/test_serialization.py +++ b/python/packages/core/tests/workflow/test_serialization.py @@ -799,3 +799,48 @@ def test_comprehensive_edge_groups_workflow_serialization() -> None: assert len(fan_in_groups[0]["edges"]) == 2, "FanInEdgeGroup should have 2 edges (from parallel_1 and parallel_2)" for single_group in single_groups: assert len(single_group["edges"]) == 1, "Each SingleEdgeGroup should have exactly 1 edge" + + +def test_to_dict_preserves_compatibility_wire_keys_for_output_designation() -> None: + """to_dict() must emit the compatibility wire keys regardless of the Python kwarg names. + + The Python API renamed ``output_executors`` -> ``output_from`` and + uses ``intermediate_output_from`` for intermediate selection, but the serialized + dict must keep the old keys so existing checkpoints stay readable. This is a + regression guard against accidental renames of the wire format. + """ + + class _Yielder(Executor): + @handler + async def handle(self, message: str, ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output(message) + await ctx.send_message(message) + + class _Terminal(Executor): + @handler + async def handle(self, message: str, ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output(f"final: {message}") + + start = _Yielder(id="start") + progress = _Yielder(id="progress") + final = _Terminal(id="final") + + workflow = ( + WorkflowBuilder( + start_executor=start, + output_from=[final], + intermediate_output_from=[progress], + ) + .add_edge(start, progress) + .add_edge(progress, final) + .build() + ) + + d = workflow.to_dict() + + assert "output_executors" in d, "wire key 'output_executors' must be preserved" + assert "intermediate_executors" in d, "wire key 'intermediate_executors' must be preserved" + assert "output_from" not in d, "new Python kwarg name must NOT leak into the wire format" + assert "intermediate_output_from" not in d, "new Python kwarg name must NOT leak into the wire format" + assert d["output_executors"] == ["final"] + assert d["intermediate_executors"] == ["progress"] diff --git a/python/packages/core/tests/workflow/test_strict_mode_event_labeling.py b/python/packages/core/tests/workflow/test_strict_mode_event_labeling.py new file mode 100644 index 0000000000..d1de5c3cb0 --- /dev/null +++ b/python/packages/core/tests/workflow/test_strict_mode_event_labeling.py @@ -0,0 +1,118 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for the runner's explicit output selection event labeling.""" + +from __future__ import annotations + +import warnings +from typing import Any + +import pytest +from typing_extensions import Never + +from agent_framework import ( + Message, + WorkflowBuilder, + WorkflowContext, + executor, +) + + +@executor +async def _start(messages: list[Message], ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output("from-start") + await ctx.send_message("downstream") + + +@executor +async def _downstream(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("from-downstream") + + +def _input_msg() -> list[Message]: + return [Message(role="user", contents=["hi"])] + + +@pytest.mark.asyncio +async def test_strict_mode_designated_executor_emits_output_events() -> None: + """Output-designated executor yields produce type='output' events.""" + workflow = WorkflowBuilder(start_executor=_start, output_from=[_start]).add_edge(_start, _downstream).build() + output_events: list[Any] = [] + intermediate_events: list[Any] = [] + async for event in workflow.run(_input_msg(), stream=True): + if event.type == "output": + output_events.append(event) + elif event.type == "intermediate": + intermediate_events.append(event) + + assert any(ev.data == "from-start" for ev in output_events), "designated executor's yield is type='output'" + assert intermediate_events == [] + assert all(ev.data != "from-downstream" for ev in output_events), "unlisted executor yield is hidden" + + +@pytest.mark.asyncio +async def test_intermediate_designated_executor_emits_intermediate_events() -> None: + """Intermediate-designated executor yields produce type='intermediate' events.""" + workflow = ( + WorkflowBuilder(start_executor=_start, intermediate_output_from=[_downstream]) + .add_edge(_start, _downstream) + .build() + ) + output_events: list[Any] = [] + intermediate_events: list[Any] = [] + async for event in workflow.run(_input_msg(), stream=True): + if event.type == "output": + output_events.append(event) + elif event.type == "intermediate": + intermediate_events.append(event) + + assert len(output_events) == 0 + assert {ev.data for ev in intermediate_events} == {"from-downstream"} + + +@pytest.mark.asyncio +async def test_omitted_selection_keeps_all_yields_as_output() -> None: + """Omitted output selection preserves today's behavior: all yields are type='output'.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + workflow = WorkflowBuilder(start_executor=_start).add_edge(_start, _downstream).build() + output_events: list[Any] = [] + intermediate_events: list[Any] = [] + async for event in workflow.run(_input_msg(), stream=True): + if event.type == "output": + output_events.append(event) + elif event.type == "intermediate": + intermediate_events.append(event) + + assert {ev.data for ev in output_events} == {"from-start", "from-downstream"} + assert len(intermediate_events) == 0 + + +@pytest.mark.asyncio +async def test_strict_mode_get_outputs_returns_only_designated() -> None: + """WorkflowRunResult.get_outputs() returns only output-designated payloads.""" + workflow = ( + WorkflowBuilder( + start_executor=_start, + output_from=[_downstream], + intermediate_output_from=[_start], + ) + .add_edge(_start, _downstream) + .build() + ) + result = await workflow.run(_input_msg()) + assert result.get_outputs() == ["from-downstream"] + assert result.get_intermediate_outputs() == ["from-start"] + + +@pytest.mark.asyncio +async def test_hidden_yields_remain_in_executor_completion_events() -> None: + """Hidden yield_output payloads stay available through executor_completed observability.""" + workflow = WorkflowBuilder(start_executor=_start, output_from=[_downstream]).add_edge(_start, _downstream).build() + result = await workflow.run(_input_msg()) + assert result.get_outputs() == ["from-downstream"] + assert result.get_intermediate_outputs() == [] + assert not any(event.type in {"output", "intermediate"} and event.data == "from-start" for event in result) + completed = [event for event in result if event.type == "executor_completed" and event.executor_id == _start.id] + assert completed + assert completed[0].data == ["downstream", "from-start"] diff --git a/python/packages/core/tests/workflow/test_sub_workflow.py b/python/packages/core/tests/workflow/test_sub_workflow.py index 666e82f4d7..7bf38a06f3 100644 --- a/python/packages/core/tests/workflow/test_sub_workflow.py +++ b/python/packages/core/tests/workflow/test_sub_workflow.py @@ -617,3 +617,75 @@ async def test_sub_workflow_checkpoint_restore_no_duplicate_requests() -> None: # Key assertion: Only the second request should be received, not a duplicate of the first assert len(request_events) == 1 assert request_events[0].data.prompt == "Second request" + + +async def test_sub_workflow_intermediate_outputs_propagate_to_parent() -> None: + """A child workflow's intermediate emissions must bubble up through the parent. + + Regression guard for the bug where WorkflowExecutor._process_workflow_result only + forwarded result.get_outputs() and silently dropped result.get_intermediate_outputs(). + The forwarded event must carry the WorkflowExecutor's own id as the source so outer + callers don't have to know the child's internal executor layout, and it must keep + type='intermediate' regardless of how the parent designates the WorkflowExecutor. + """ + + class _ProgressEmitter(Executor): + def __init__(self) -> None: + super().__init__(id="progress_emitter") + + @handler + async def run(self, message: str, ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output(f"progress: {message}") + await ctx.send_message(message) + + class _Finalizer(Executor): + def __init__(self) -> None: + super().__init__(id="finalizer") + + @handler + async def run(self, message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output(f"final: {message}") + + progress = _ProgressEmitter() + finalizer = _Finalizer() + child = ( + WorkflowBuilder( + start_executor=progress, + output_from=[finalizer], + intermediate_output_from=[progress], + ) + .add_edge(progress, finalizer) + .build() + ) + + sub = WorkflowExecutor(child, id="sub") + + class _ParentSink(Executor): + def __init__(self) -> None: + super().__init__(id="parent_sink") + self.received: list[str] = [] + + @handler + async def run(self, message: str, ctx: WorkflowContext[Never, str]) -> None: + self.received.append(message) + await ctx.yield_output(message) + + sink = _ParentSink() + parent = WorkflowBuilder(start_executor=sub, output_from=[sink]).add_edge(sub, sink).build() + + intermediate_events: list[WorkflowEvent[Any]] = [] + output_events: list[WorkflowEvent[Any]] = [] + async for event in parent.run("hello", stream=True): + if event.type == "intermediate": + intermediate_events.append(event) + elif event.type == "output": + output_events.append(event) + + # The child's intermediate emission bubbled up labeled with the WorkflowExecutor id, + # not the child's internal executor id. + assert len(intermediate_events) == 1, [(e.executor_id, e.data) for e in intermediate_events] + assert intermediate_events[0].executor_id == "sub" + assert intermediate_events[0].data == "progress: hello" + + # The parent's own terminal output is unaffected. + assert any(e.executor_id == "parent_sink" and e.data == "final: hello" for e in output_events) diff --git a/python/packages/core/tests/workflow/test_validation.py b/python/packages/core/tests/workflow/test_validation.py index be3c8b45f7..a9f62f35a3 100644 --- a/python/packages/core/tests/workflow/test_validation.py +++ b/python/packages/core/tests/workflow/test_validation.py @@ -550,12 +550,10 @@ def test_output_validation_with_valid_output_executors(): executor2 = OutputExecutor(id="executor2") # Build workflow with valid output executors - workflow = ( - WorkflowBuilder(start_executor=executor1, output_executors=[executor2]).add_edge(executor1, executor2).build() - ) + workflow = WorkflowBuilder(start_executor=executor1, output_from=[executor2]).add_edge(executor1, executor2).build() assert workflow is not None - assert workflow._output_executors == ["executor2"] # pyright: ignore[reportPrivateUsage] + assert {ex.id for ex in workflow.get_output_executors()} == {"executor2"} def test_output_validation_with_multiple_valid_output_executors(): @@ -565,14 +563,14 @@ def test_output_validation_with_multiple_valid_output_executors(): executor3 = OutputExecutor(id="executor3") workflow = ( - WorkflowBuilder(start_executor=executor1, output_executors=[executor1, executor3]) + WorkflowBuilder(start_executor=executor1, output_from=[executor1, executor3]) .add_edge(executor1, executor2) .add_edge(executor2, executor3) .build() ) assert workflow is not None - assert set(workflow._output_executors) == {"executor1", "executor3"} # pyright: ignore[reportPrivateUsage] + assert {ex.id for ex in workflow.get_output_executors()} == {"executor1", "executor3"} def test_output_validation_fails_for_nonexistent_executor(): @@ -598,7 +596,7 @@ def test_output_validation_fails_for_executor_without_output_types(): with pytest.raises(WorkflowValidationError) as exc_info: ( - WorkflowBuilder(start_executor=executor1, output_executors=[no_output_executor]) + WorkflowBuilder(start_executor=executor1, output_from=[no_output_executor]) .add_edge(executor1, no_output_executor) .build() ) @@ -608,16 +606,77 @@ def test_output_validation_fails_for_executor_without_output_types(): assert exc_info.value.validation_type == ValidationTypeEnum.OUTPUT_VALIDATION -def test_output_validation_empty_list_passes(): - """Test that output validation passes with an empty output executors list.""" +def test_output_validation_empty_explicit_designation_fails(): + """Test that explicit mode rejects an empty output/intermediate designation.""" executor1 = OutputExecutor(id="executor1") executor2 = OutputExecutor(id="executor2") - workflow = WorkflowBuilder(start_executor=executor1, output_executors=[]).add_edge(executor1, executor2).build() + with pytest.raises(WorkflowValidationError) as exc_info: + WorkflowBuilder(start_executor=executor1, output_from=[]).add_edge(executor1, executor2).build() + + assert "at least one output or intermediate executor" in str(exc_info.value) + assert exc_info.value.validation_type == ValidationTypeEnum.OUTPUT_VALIDATION + + +def test_output_validation_with_valid_intermediate_executors(): + """Test that output validation passes when intermediate executors exist and have output types.""" + executor1 = OutputExecutor(id="executor1") + executor2 = OutputExecutor(id="executor2") + + workflow = ( + WorkflowBuilder(start_executor=executor1, intermediate_output_from=[executor1]) + .add_edge(executor1, executor2) + .build() + ) assert workflow is not None - # All executors are outputs - assert workflow._output_executors == ["executor1", "executor2"] # type: ignore + assert {ex.id for ex in workflow.get_intermediate_executors()} == {"executor1"} + assert workflow.is_intermediate_executor("executor1") + assert not workflow.is_terminal_executor("executor2") + + +def test_output_validation_fails_for_designation_overlap(): + """Test that an executor cannot be both terminal and intermediate.""" + executor1 = OutputExecutor(id="executor1") + + with pytest.raises(WorkflowValidationError) as exc_info: + WorkflowBuilder( + start_executor=executor1, + output_from=[executor1], + intermediate_output_from=[executor1], + ).build() + + assert "both output and intermediate" in str(exc_info.value) + assert exc_info.value.validation_type == ValidationTypeEnum.OUTPUT_VALIDATION + + +def test_output_validation_fails_for_duplicate_designation(): + """Test that duplicate output or intermediate designation entries are rejected.""" + executor1 = OutputExecutor(id="executor1") + + with pytest.raises(WorkflowValidationError) as exc_info: + WorkflowBuilder(start_executor=executor1, output_from=[executor1, executor1]).build() + + assert "Duplicate output executor designation" in str(exc_info.value) + assert exc_info.value.validation_type == ValidationTypeEnum.OUTPUT_VALIDATION + + +def test_output_validation_fails_for_unknown_intermediate_executor(): + """Test that intermediate designation rejects executors outside the workflow graph.""" + executor1 = OutputExecutor(id="executor1") + executor2 = OutputExecutor(id="executor2") + missing = OutputExecutor(id="missing") + + with pytest.raises(WorkflowValidationError) as exc_info: + ( + WorkflowBuilder(start_executor=executor1, intermediate_output_from=[missing]) + .add_edge(executor1, executor2) + .build() + ) + + assert "not present in the workflow graph" in str(exc_info.value) + assert "missing" in str(exc_info.value) + assert exc_info.value.validation_type == ValidationTypeEnum.OUTPUT_VALIDATION def test_output_validation_with_direct_validate_workflow_graph(): diff --git a/python/packages/core/tests/workflow/test_workflow.py b/python/packages/core/tests/workflow/test_workflow.py index 30e81d8fe6..27f24d26f9 100644 --- a/python/packages/core/tests/workflow/test_workflow.py +++ b/python/packages/core/tests/workflow/test_workflow.py @@ -1056,7 +1056,7 @@ class PassthroughExecutor(Executor): async def test_output_executors_empty_yields_all_outputs() -> None: - """Test that when _output_executors is empty (default), all outputs are yielded.""" + """Test that omitted output selection yields all outputs for compatibility.""" # Create executors that each produce different outputs executor_a = PassthroughExecutor(id="executor_a", output_value=10) executor_b = OutputProducerExecutor(id="executor_b", output_value=20) @@ -1085,9 +1085,7 @@ async def test_output_executors_filters_outputs_non_streaming() -> None: # Build workflow with a -> b workflow = ( - WorkflowBuilder(start_executor=executor_a, output_executors=[executor_b]) - .add_edge(executor_a, executor_b) - .build() + WorkflowBuilder(start_executor=executor_a, output_from=[executor_b]).add_edge(executor_a, executor_b).build() ) result = await workflow.run(NumberMessage(data=0)) @@ -1110,9 +1108,7 @@ async def test_output_executors_filters_outputs_streaming() -> None: # Build workflow with a -> b workflow = ( - WorkflowBuilder(start_executor=executor_a, output_executors=[executor_a]) - .add_edge(executor_a, executor_b) - .build() + WorkflowBuilder(start_executor=executor_a, output_from=[executor_a]).add_edge(executor_a, executor_b).build() ) # Collect outputs from streaming @@ -1136,7 +1132,7 @@ async def test_output_executors_with_multiple_specified_executors() -> None: # Build workflow with a -> b -> c workflow = ( - WorkflowBuilder(start_executor=executor_a, output_executors=[executor_a, executor_c]) + WorkflowBuilder(start_executor=executor_a, output_from=[executor_a, executor_c]) .add_edge(executor_a, executor_b) .add_edge(executor_b, executor_c) .build() @@ -1154,12 +1150,15 @@ async def test_output_executors_with_multiple_specified_executors() -> None: async def test_output_executors_with_nonexistent_executor_id() -> None: """Test that specifying a non-existent executor ID doesn't break the workflow.""" + from agent_framework._workflows._workflow import OutputDesignation + executor_a = OutputProducerExecutor(id="executor_a", output_value=42) workflow = WorkflowBuilder(start_executor=executor_a).build() - # Set output_executors to an ID that doesn't exist - workflow._output_executors = ["nonexistent_executor"] # type: ignore + # Designate a nonexistent executor so the workflow-level filter drops every yield. + workflow._output_designation = OutputDesignation(outputs=frozenset({"nonexistent_executor"})) # type: ignore[attr-defined] + workflow._runner.context.set_yield_output_classifier(workflow._output_designation.classify) # type: ignore[attr-defined,reportPrivateUsage] result = await workflow.run(NumberMessage(data=0)) outputs = result.get_outputs() @@ -1199,7 +1198,7 @@ async def test_output_executors_filtering_with_fan_in() -> None: # Build fan-in workflow: start -> [a, b] -> aggregator workflow = ( - WorkflowBuilder(start_executor=executor_start, output_executors=[aggregator]) + WorkflowBuilder(start_executor=executor_start, output_from=[aggregator]) .add_fan_out_edges(executor_start, [executor_a, executor_b]) .add_fan_in_edges([executor_a, executor_b], aggregator) .build() @@ -1218,7 +1217,7 @@ async def test_output_executors_filtering_with_run_responses() -> None: """Test output filtering works correctly with run(responses=...) method.""" executor = MockExecutorRequestApproval(id="approval_executor") - workflow = WorkflowBuilder(start_executor=executor, output_executors=[executor]).build() + workflow = WorkflowBuilder(start_executor=executor, output_from=[executor]).build() # Run workflow which will request approval result = await workflow.run(NumberMessage(data=42)) @@ -1252,8 +1251,11 @@ async def test_output_executors_filtering_with_run_responses_streaming() -> None request_events = [e for e in events_list if e.type == "request_info"] assert len(request_events) == 1 - # Set output_executors to exclude the approval executor - workflow._output_executors = ["other_executor"] # type: ignore + # Designate a different executor so the workflow-level filter drops the approval yield. + from agent_framework._workflows._workflow import OutputDesignation + + workflow._output_designation = OutputDesignation(outputs=frozenset({"other_executor"})) # type: ignore[attr-defined] + workflow._runner.context.set_yield_output_classifier(workflow._output_designation.classify) # type: ignore[attr-defined,reportPrivateUsage] # Send approval response via streaming responses = {request_events[0].request_id: ApprovalMessage(approved=True)} diff --git a/python/packages/core/tests/workflow/test_workflow_agent.py b/python/packages/core/tests/workflow/test_workflow_agent.py index 0101a6e8a5..3dcdd26c86 100644 --- a/python/packages/core/tests/workflow/test_workflow_agent.py +++ b/python/packages/core/tests/workflow/test_workflow_agent.py @@ -923,7 +923,7 @@ class TestWorkflowAgent: # Build workflow: start -> agent1 (no output) -> agent2 (output visible) workflow = ( - WorkflowBuilder(start_executor=start_exec, output_executors=[start_exec, agent2]) + WorkflowBuilder(start_executor=start_exec, output_from=[start_exec, agent2]) .add_edge(start_exec, agent1) .add_edge(agent1, agent2) .build() diff --git a/python/packages/core/tests/workflow/test_workflow_agent_intermediate.py b/python/packages/core/tests/workflow/test_workflow_agent_intermediate.py new file mode 100644 index 0000000000..4fc66135f5 --- /dev/null +++ b/python/packages/core/tests/workflow/test_workflow_agent_intermediate.py @@ -0,0 +1,353 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for WorkflowAgent forwarding of intermediate workflow events. + +Covers: +- type='intermediate' surfaces as AgentResponseUpdate without content-type rewriting +- type='data' (compatibility alias via WorkflowEvent.emit) is forwarded +- Message.additional_properties survives the intermediate translation path +- Terminal yields keep using regular text content (backward compat) +""" + +from __future__ import annotations + +import warnings + +import pytest +from typing_extensions import Never + +from agent_framework import ( + AgentResponse, + AgentResponseUpdate, + Content, + Message, + WorkflowBuilder, + WorkflowContext, + WorkflowEvent, + executor, +) +from agent_framework.exceptions import AgentInvalidRequestException + + +@pytest.mark.asyncio +async def test_workflow_agent_forwards_intermediate_events_without_content_rewrite() -> None: + """An intermediate yield from an intermediate-designated executor surfaces through as_agent + as an AgentResponseUpdate carrying its original content type.""" + + @executor + async def emit(messages: list[Message], ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output("intermediate progress") + await ctx.send_message("downstream") + + @executor + async def terminal(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("FINAL") + + workflow = ( + WorkflowBuilder( + start_executor=emit, + output_from=[terminal], + intermediate_output_from=[emit], + ) + .add_edge(emit, terminal) + .build() + ) + agent = workflow.as_agent("test") + + updates: list[AgentResponseUpdate] = [] + async for update in agent.run("hi", stream=True): + updates.append(update) + + text = " ".join(c.text for u in updates for c in u.contents if c.type == "text") + reasoning_text = " ".join(c.text for u in updates for c in u.contents if c.type == "text_reasoning") + + assert "intermediate progress" in text + assert "FINAL" in text + assert reasoning_text == "" + + +@pytest.mark.asyncio +async def test_workflow_agent_text_accessor_includes_forwarded_intermediate_text() -> None: + """Intermediate text is forwarded as text until issue 5885 defines the final mapping.""" + + @executor + async def emit(messages: list[Message], ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output("invisible-progress") + await ctx.send_message("forward") + + @executor + async def terminal(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("the-answer") + + workflow = ( + WorkflowBuilder( + start_executor=emit, + output_from=[terminal], + intermediate_output_from=[emit], + ) + .add_edge(emit, terminal) + .build() + ) + agent = workflow.as_agent("test") + + response = await agent.run("hi") + assert isinstance(response, AgentResponse) + assert "invisible-progress" in response.text + assert "the-answer" in response.text + + +@pytest.mark.asyncio +async def test_workflow_agent_hidden_yields_do_not_surface_non_streaming() -> None: + """In explicit designation mode, unlisted executor yields stay out of agent responses.""" + + @executor + async def hidden(messages: list[Message], ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output("hidden-progress") + await ctx.send_message("forward") + + @executor + async def terminal(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("visible-answer") + + workflow = WorkflowBuilder(start_executor=hidden, output_from=[terminal]).add_edge(hidden, terminal).build() + agent = workflow.as_agent("test") + + response = await agent.run("hi") + all_text = " ".join(c.text for m in response.messages for c in m.contents if hasattr(c, "text")) + + assert response.text == "visible-answer" + assert "hidden-progress" not in all_text + + +@pytest.mark.asyncio +async def test_workflow_agent_hidden_yields_do_not_surface_streaming() -> None: + """In explicit designation mode, unlisted executor yields stay out of agent updates.""" + + @executor + async def hidden(messages: list[Message], ctx: WorkflowContext[str, str]) -> None: + await ctx.yield_output("hidden-progress") + await ctx.send_message("forward") + + @executor + async def terminal(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("visible-answer") + + workflow = WorkflowBuilder(start_executor=hidden, output_from=[terminal]).add_edge(hidden, terminal).build() + agent = workflow.as_agent("test") + + updates: list[AgentResponseUpdate] = [] + async for update in agent.run("hi", stream=True): + updates.append(update) + + all_text = " ".join(c.text for u in updates for c in u.contents if hasattr(c, "text")) + + assert "visible-answer" in all_text + assert "hidden-progress" not in all_text + + +@pytest.mark.asyncio +async def test_workflow_agent_data_event_emit_factory_still_forwarded() -> None: + """Even the deprecated WorkflowEvent.emit() / type='data' path is forwarded.""" + + @executor + async def emit_data_alias(messages: list[Message], ctx: WorkflowContext[Never, str]) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + await ctx.add_event(WorkflowEvent.emit("emit_data_alias", "data-alias-payload")) + await ctx.yield_output("DONE") + + workflow = WorkflowBuilder(start_executor=emit_data_alias, output_from=[emit_data_alias]).build() + agent = workflow.as_agent("test") + + updates: list[AgentResponseUpdate] = [] + async for update in agent.run("hi", stream=True): + updates.append(update) + + text = " ".join(c.text for u in updates for c in u.contents if c.type == "text") + assert "data-alias-payload" in text + + +@pytest.mark.asyncio +async def test_workflow_agent_intermediate_message_preserves_additional_properties() -> None: + """Message.additional_properties survives intermediate forwarding. + + Producer-attached metadata (tracking_id, conversation_id, etc.) must not disappear + for messages flowing through intermediate-designated executors. + """ + + @executor + async def emit(messages: list[Message], ctx: WorkflowContext[str, AgentResponse]) -> None: + msg = Message( + role="assistant", + contents=[Content.from_text(text="hi")], + additional_properties={"tracking_id": "abc-123"}, + ) + await ctx.yield_output(AgentResponse(messages=[msg])) + await ctx.send_message("forward") + + @executor + async def terminal(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("done") + + workflow = ( + WorkflowBuilder( + start_executor=emit, + output_from=[terminal], + intermediate_output_from=[emit], + ) + .add_edge(emit, terminal) + .build() + ) + agent = workflow.as_agent("test") + + response = await agent.run("hi") + intermediate_msgs = [m for m in response.messages if any(c.type == "text" and c.text == "hi" for c in m.contents)] + assert intermediate_msgs, "expected at least one intermediate message in the response" + assert intermediate_msgs[0].additional_properties.get("tracking_id") == "abc-123" + + +@pytest.mark.asyncio +async def test_workflow_agent_terminal_text_stays_text_not_reasoning() -> None: + """A designated executor's text yield surfaces as Content.text.""" + + @executor + async def only(messages: list[Message], ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("the-answer") + + workflow = WorkflowBuilder(start_executor=only, output_from=[only]).build() + agent = workflow.as_agent("test") + + response = await agent.run("hi") + assert response.text == "the-answer" + # No text_reasoning content because everything from `only` is terminal. + assert all(c.type != "text_reasoning" for m in response.messages for c in m.contents) + + +@pytest.mark.asyncio +async def test_workflow_agent_non_streaming_rejects_terminal_update() -> None: + """A terminal event carrying AgentResponseUpdate is streaming-only and invalid in run().""" + + @executor + async def emit(messages: list[Message], ctx: WorkflowContext[Never, AgentResponseUpdate]) -> None: + await ctx.yield_output(AgentResponseUpdate(contents=[Content.from_text(text="partial")], role="assistant")) + + workflow = WorkflowBuilder(start_executor=emit, output_from=[emit]).build() + agent = workflow.as_agent("test") + + with pytest.raises(AgentInvalidRequestException, match="AgentResponseUpdate"): + await agent.run("hi") + + +@pytest.mark.asyncio +async def test_workflow_agent_non_streaming_rejects_intermediate_update() -> None: + """An intermediate event carrying AgentResponseUpdate is streaming-only and invalid in run().""" + + @executor + async def emit(messages: list[Message], ctx: WorkflowContext[str, AgentResponseUpdate]) -> None: + await ctx.yield_output(AgentResponseUpdate(contents=[Content.from_text(text="partial")], role="assistant")) + await ctx.send_message("forward") + + @executor + async def terminal(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output("FINAL") + + workflow = ( + WorkflowBuilder( + start_executor=emit, + output_from=[terminal], + intermediate_output_from=[emit], + ) + .add_edge(emit, terminal) + .build() + ) + agent = workflow.as_agent("test") + + with pytest.raises(AgentInvalidRequestException, match="AgentResponseUpdate"): + await agent.run("hi") + + +@pytest.mark.asyncio +async def test_workflow_agent_streaming_update_payloads_preserve_classification() -> None: + """Streaming AgentResponseUpdate payloads preserve original content types.""" + + @executor + async def emit(messages: list[Message], ctx: WorkflowContext[str, AgentResponseUpdate]) -> None: + await ctx.yield_output( + AgentResponseUpdate(contents=[Content.from_text(text="intermediate-chunk")], role="assistant") + ) + await ctx.send_message("forward") + + @executor + async def terminal(message: str, ctx: WorkflowContext[Never, AgentResponseUpdate]) -> None: + await ctx.yield_output( + AgentResponseUpdate(contents=[Content.from_text(text="terminal-chunk")], role="assistant") + ) + + workflow = ( + WorkflowBuilder( + start_executor=emit, + output_from=[terminal], + intermediate_output_from=[emit], + ) + .add_edge(emit, terminal) + .build() + ) + agent = workflow.as_agent("test") + + updates: list[AgentResponseUpdate] = [] + async for update in agent.run("hi", stream=True): + updates.append(update) + + text = " ".join(c.text for u in updates for c in u.contents if c.type == "text") + reasoning_text = " ".join(c.text for u in updates for c in u.contents if c.type == "text_reasoning") + + assert "intermediate-chunk" in text + assert "terminal-chunk" in text + assert reasoning_text == "" + + +@pytest.mark.asyncio +async def test_workflow_agent_drops_orchestration_internal_events() -> None: + """Orchestration-internal event types (group_chat / handoff_sent / magentic_orchestrator) + must not surface through workflow.as_agent(). Their dataclass payloads would otherwise + be stringified by the generic fallback path and leak into response history.""" + + @executor + async def emit(messages: list[Message], ctx: WorkflowContext[Never, str]) -> None: + # Construct typed orchestration-internal events directly to assert they get + # dropped at the agent boundary regardless of payload. + await ctx.add_event(WorkflowEvent("group_chat", data={"orchestrator": "details"})) # type: ignore[arg-type] + await ctx.add_event(WorkflowEvent("handoff_sent", data={"target": "agent_b"})) # type: ignore[arg-type] + await ctx.add_event(WorkflowEvent("magentic_orchestrator", data={"plan": "..."})) # type: ignore[arg-type] + await ctx.yield_output("FINAL") + + workflow = WorkflowBuilder(start_executor=emit, output_from=[emit]).build() + agent = workflow.as_agent("test") + + response = await agent.run("hi") + all_text = " ".join(c.text for m in response.messages for c in m.contents if hasattr(c, "text")) + assert "orchestrator" not in all_text + assert "agent_b" not in all_text + assert "plan" not in all_text + assert response.text == "FINAL" + + +@pytest.mark.asyncio +async def test_workflow_agent_drops_orchestration_internal_events_streaming() -> None: + """Streaming counterpart — orchestration-internal events stay inside the workflow.""" + + @executor + async def emit(messages: list[Message], ctx: WorkflowContext[Never, str]) -> None: + await ctx.add_event(WorkflowEvent("group_chat", data={"orchestrator": "details"})) # type: ignore[arg-type] + await ctx.yield_output("FINAL") + + workflow = WorkflowBuilder(start_executor=emit, output_from=[emit]).build() + agent = workflow.as_agent("test") + + updates: list[AgentResponseUpdate] = [] + async for update in agent.run("hi", stream=True): + updates.append(update) + + all_text = " ".join(c.text for u in updates for c in u.contents if hasattr(c, "text")) + assert "orchestrator" not in all_text + assert "FINAL" in all_text diff --git a/python/packages/core/tests/workflow/test_workflow_builder.py b/python/packages/core/tests/workflow/test_workflow_builder.py index c780bf4ac9..873d6e7c73 100644 --- a/python/packages/core/tests/workflow/test_workflow_builder.py +++ b/python/packages/core/tests/workflow/test_workflow_builder.py @@ -254,10 +254,10 @@ def test_switch_case_with_agents(): def test_with_output_from_returns_builder(): """Test that with_output_from returns the builder for method chaining.""" executor_a = MockExecutor(id="executor_a") - builder = WorkflowBuilder(output_executors=[executor_a], start_executor=executor_a) + builder = WorkflowBuilder(output_from=[executor_a], start_executor=executor_a) - # Verify builder was created with output_executors - assert builder._output_executors == [executor_a] # pyright: ignore[reportPrivateUsage] + # Verify builder was created with output_from + assert builder._output_from == [executor_a] # pyright: ignore[reportPrivateUsage] def test_with_output_from_with_executor_instances(): @@ -266,13 +266,11 @@ def test_with_output_from_with_executor_instances(): executor_b = MockExecutor(id="executor_b") workflow = ( - WorkflowBuilder(start_executor=executor_a, output_executors=[executor_b]) - .add_edge(executor_a, executor_b) - .build() + WorkflowBuilder(start_executor=executor_a, output_from=[executor_b]).add_edge(executor_a, executor_b).build() ) # Verify that the workflow was built with the correct output executors - assert workflow._output_executors == ["executor_b"] # type: ignore + assert {ex.id for ex in workflow.get_output_executors()} == {"executor_b"} def test_with_output_from_with_agent_instances(): @@ -280,10 +278,10 @@ def test_with_output_from_with_agent_instances(): agent_a = DummyAgent(id="agent_a", name="writer") agent_b = DummyAgent(id="agent_b", name="reviewer") - workflow = WorkflowBuilder(start_executor=agent_a, output_executors=[agent_b]).add_edge(agent_a, agent_b).build() + workflow = WorkflowBuilder(start_executor=agent_a, output_from=[agent_b]).add_edge(agent_a, agent_b).build() # Verify that the workflow was built with the agent's name as output executor - assert workflow._output_executors == ["reviewer"] # type: ignore + assert {ex.id for ex in workflow.get_output_executors()} == {"reviewer"} def test_with_output_from_with_executor_instances_by_id(): @@ -292,12 +290,10 @@ def test_with_output_from_with_executor_instances_by_id(): executor_b = MockExecutor(id="ExecutorB") workflow = ( - WorkflowBuilder(start_executor=executor_a, output_executors=[executor_b]) - .add_edge(executor_a, executor_b) - .build() + WorkflowBuilder(start_executor=executor_a, output_from=[executor_b]).add_edge(executor_a, executor_b).build() ) - assert workflow._output_executors == ["ExecutorB"] # type: ignore + assert {ex.id for ex in workflow.get_output_executors()} == {"ExecutorB"} def test_with_output_from_with_multiple_executors(): @@ -307,29 +303,27 @@ def test_with_output_from_with_multiple_executors(): executor_c = MockExecutor(id="executor_c") workflow = ( - WorkflowBuilder(start_executor=executor_a, output_executors=[executor_a, executor_c]) + WorkflowBuilder(start_executor=executor_a, output_from=[executor_a, executor_c]) .add_edge(executor_a, executor_b) .add_edge(executor_b, executor_c) .build() ) # Verify that the workflow was built with both output executors - assert set(workflow._output_executors) == {"executor_a", "executor_c"} # type: ignore + assert {ex.id for ex in workflow.get_output_executors()} == {"executor_a", "executor_c"} def test_with_output_from_can_be_set_to_different_value(): - """Test that output_executors can be set at construction time.""" + """Test that output_from can be set at construction time.""" executor_a = MockExecutor(id="executor_a") executor_b = MockExecutor(id="executor_b") workflow = ( - WorkflowBuilder(start_executor=executor_a, output_executors=[executor_b]) - .add_edge(executor_a, executor_b) - .build() + WorkflowBuilder(start_executor=executor_a, output_from=[executor_b]).add_edge(executor_a, executor_b).build() ) # Verify that the setting is applied - assert workflow._output_executors == ["executor_b"] # type: ignore + assert {ex.id for ex in workflow.get_output_executors()} == {"executor_b"} def test_with_output_from_with_agent_instances_resolves_name(): @@ -338,37 +332,37 @@ def test_with_output_from_with_agent_instances_resolves_name(): agent_reviewer = DummyAgent(id="agent2", name="reviewer") workflow = ( - WorkflowBuilder(start_executor=agent_writer, output_executors=[agent_reviewer]) + WorkflowBuilder(start_executor=agent_writer, output_from=[agent_reviewer]) .add_edge(agent_writer, agent_reviewer) .build() ) - assert workflow._output_executors == ["reviewer"] # type: ignore + assert {ex.id for ex in workflow.get_output_executors()} == {"reviewer"} def test_with_output_from_in_constructor(): - """Test that output_executors works correctly when set in the constructor.""" + """Test that output_from works correctly when set in the constructor.""" executor_a = MockExecutor(id="executor_a") executor_b = MockExecutor(id="executor_b") executor_c = MockExecutor(id="executor_c") - # Build workflow with output_executors in the constructor + # Build workflow with output_from in the constructor workflow = ( - WorkflowBuilder(start_executor=executor_a, output_executors=[executor_c]) + WorkflowBuilder(start_executor=executor_a, output_from=[executor_c]) .add_edge(executor_a, executor_b) .add_edge(executor_b, executor_c) .build() ) # Verify that the setting persists through the chain - assert workflow._output_executors == ["executor_c"] # type: ignore + assert {ex.id for ex in workflow.get_output_executors()} == {"executor_c"} def test_with_output_from_with_invalid_executor_raises_validation_error(): """Test that with_output_from with an invalid executor raises an error.""" executor_a = MockExecutor(id="executor_a") - builder = WorkflowBuilder(start_executor=executor_a, output_executors=[MockExecutor(id="executor_b")]) + builder = WorkflowBuilder(start_executor=executor_a, output_from=[MockExecutor(id="executor_b")]) # Attempting to set output from an executor not in the workflow should raise an error with pytest.raises( diff --git a/python/packages/core/tests/workflow/test_workflow_context.py b/python/packages/core/tests/workflow/test_workflow_context.py index a13c0b5a55..5889892435 100644 --- a/python/packages/core/tests/workflow/test_workflow_context.py +++ b/python/packages/core/tests/workflow/test_workflow_context.py @@ -5,6 +5,7 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Any +import pytest from typing_extensions import Never from agent_framework import ( @@ -72,6 +73,31 @@ async def test_executor_cannot_emit_framework_lifecycle_event(caplog: "LogCaptur assert any("attempted to emit" in message and "'status'" in message for message in list(caplog.messages)) +@pytest.mark.parametrize( + "event", + [ + WorkflowEvent("output", executor_id="exec", data="output-payload"), + WorkflowEvent("intermediate", executor_id="exec", data="intermediate-payload"), + ], +) +async def test_executor_cannot_emit_output_selection_events( + event: WorkflowEvent[Any], + caplog: "LogCaptureFixture", +) -> None: + async with make_context() as (ctx, runner_ctx): + caplog.clear() + with caplog.at_level("WARNING"): + await ctx.add_event(event) + + events: list[WorkflowEvent] = await runner_ctx.drain_events() + assert len(events) == 1 + assert events[0].type == "warning" + data = events[0].data + assert isinstance(data, str) + assert "reserved for ctx.yield_output()" in data + assert event.data not in [emitted.data for emitted in events] + + async def test_executor_emits_normal_event() -> None: async with make_context() as (ctx, runner_ctx): # Create a normal event to test event emission diff --git a/python/packages/core/tests/workflow/test_workflow_event_factories.py b/python/packages/core/tests/workflow/test_workflow_event_factories.py new file mode 100644 index 0000000000..ad24af693b --- /dev/null +++ b/python/packages/core/tests/workflow/test_workflow_event_factories.py @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for WorkflowEvent factory methods and WorkflowEvent.emit() deprecation.""" + +from __future__ import annotations + +import warnings + +import pytest + +from agent_framework import AgentResponse, Message +from agent_framework._workflows._events import WorkflowEvent + + +def test_workflow_event_output_selection_factories_are_not_public() -> None: + """Callers should use ctx.yield_output(), not direct output/intermediate factories.""" + assert not hasattr(WorkflowEvent, "output") + assert not hasattr(WorkflowEvent, "intermediate") + + +def test_workflow_event_emit_emits_deprecation_warning() -> None: + """Calling WorkflowEvent.emit() raises a DeprecationWarning recommending the new path.""" + response = AgentResponse(messages=[Message(role="assistant", contents=["x"])]) + with pytest.warns(DeprecationWarning, match="yield_output"): + WorkflowEvent.emit(executor_id="t", data=response) + + +def test_workflow_event_emit_still_returns_data_event() -> None: + """During the deprecation window, emit() still produces a type='data' event.""" + response = AgentResponse(messages=[Message(role="assistant", contents=["x"])]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + event = WorkflowEvent.emit(executor_id="t", data=response) + assert event.type == "data" diff --git a/python/packages/core/tests/workflow/test_workflow_kwargs.py b/python/packages/core/tests/workflow/test_workflow_kwargs.py index 9c664c6ac2..7bfa47a79f 100644 --- a/python/packages/core/tests/workflow/test_workflow_kwargs.py +++ b/python/packages/core/tests/workflow/test_workflow_kwargs.py @@ -377,7 +377,7 @@ async def test_kwargs_preserved_on_response_continuation() -> None: from agent_framework import WorkflowBuilder agent = _ApprovalCapturingAgent() - workflow = WorkflowBuilder(start_executor=agent, output_executors=[agent]).build() + workflow = WorkflowBuilder(start_executor=agent, output_from=[agent]).build() # Initial run with function_invocation_kwargs — workflow should pause for approval fi_kwargs = {"token": "abc"} diff --git a/python/packages/devui/agent_framework_devui/_mapper.py b/python/packages/devui/agent_framework_devui/_mapper.py index d4529875e5..f6a52ae945 100644 --- a/python/packages/devui/agent_framework_devui/_mapper.py +++ b/python/packages/devui/agent_framework_devui/_mapper.py @@ -72,6 +72,17 @@ def _stringify_name(value: Any) -> str: return value if isinstance(value, str) else str(value) +def _workflow_output_metadata(event_type: Any, executor_id: Any) -> dict[str, Any] | None: + """Return metadata that preserves workflow yield designation on visible output.""" + if event_type not in ("output", "intermediate", "data"): + return None + return { + "workflow_event_type": event_type, + "workflow_output_kind": "terminal" if event_type == "output" else "intermediate", + "executor_id": executor_id, + } + + def _serialize_content_recursive(value: Any) -> Any: """Recursively serialize Agent Framework Content objects to JSON-compatible values. @@ -200,15 +211,21 @@ class MessageMapper: try: from agent_framework import AgentResponse, AgentResponseUpdate, WorkflowEvent - # Handle WorkflowEvent with type='output' or 'data' wrapping AgentResponseUpdate - # This must be checked BEFORE generic WorkflowEvent check - # Note: AgentExecutor uses type='output' for streaming updates - if isinstance(raw_event, WorkflowEvent) and raw_event.type in ("output", "data"): + # Handle WorkflowEvent with type='output', 'intermediate', or 'data' wrapping + # AgentResponseUpdate. This must be checked BEFORE generic WorkflowEvent check. + # Note: AgentExecutor uses type='output' for streaming updates from designated + # executors and type='intermediate' from non-designated executors. type='data' + # is the deprecated legacy variant retained for backward compat. + if isinstance(raw_event, WorkflowEvent) and raw_event.type in ("output", "intermediate", "data"): event_data = getattr(cast(Any, raw_event), "data", None) if isinstance(event_data, AgentResponseUpdate): # Preserve executor_id in context for proper output routing context["current_executor_id"] = getattr(cast(Any, raw_event), "executor_id", None) - return await self._convert_agent_update(event_data, context) + context["current_workflow_event_type"] = raw_event.type + try: + return await self._convert_agent_update(event_data, context) + finally: + context.pop("current_workflow_event_type", None) # Handle complete agent response (AgentResponse) - for non-streaming agent execution if isinstance(raw_event, AgentResponse): @@ -633,6 +650,13 @@ class MessageMapper: # Check if we're in an executor context with an existing item executor_id = context.get("current_executor_id") executor_item_key = f"exec_item_{executor_id}" if executor_id else None + workflow_metadata = _workflow_output_metadata(context.get("current_workflow_event_type"), executor_id) + + if has_text_content and workflow_metadata is not None: + current_metadata = context.get("current_message_workflow_metadata") + if current_metadata != workflow_metadata: + context.pop("current_message_id", None) + context["current_message_workflow_metadata"] = workflow_metadata # If we have an executor item, use it for deltas instead of creating a message if has_text_content and executor_item_key and executor_item_key in context: @@ -644,6 +668,15 @@ class MessageMapper: message_id = f"msg_{uuid4().hex[:8]}" context["current_message_id"] = message_id context["output_index"] = context.get("output_index", -1) + 1 + message_item = ResponseOutputMessage( + type="message", + id=message_id, + role="assistant", + content=[], + status="in_progress", + ) + if workflow_metadata is not None: + cast(Any, message_item).metadata = workflow_metadata # Add message output item events.append( @@ -651,9 +684,7 @@ class MessageMapper: type="response.output_item.added", output_index=context["output_index"], sequence_number=self._next_sequence(context), - item=ResponseOutputMessage( - type="message", id=message_id, role="assistant", content=[], status="in_progress" - ), + item=message_item, ) ) @@ -675,17 +706,18 @@ class MessageMapper: # Special handling for TextContent to use proper delta events if content.type == "text" and "current_message_id" in context: # Stream text content via proper delta events - events.append( - ResponseTextDeltaEvent( - type="response.output_text.delta", - output_index=context["output_index"], - content_index=context.get("content_index", 0), - item_id=context["current_message_id"], - delta=content.text, - logprobs=[], # We don't have logprobs from Agent Framework - sequence_number=self._next_sequence(context), - ) + delta_event = ResponseTextDeltaEvent( + type="response.output_text.delta", + output_index=context["output_index"], + content_index=context.get("content_index", 0), + item_id=context["current_message_id"], + delta=content.text, + logprobs=[], # We don't have logprobs from Agent Framework + sequence_number=self._next_sequence(context), ) + if workflow_metadata is not None: + cast(Any, delta_event).metadata = workflow_metadata + events.append(delta_event) elif content.type in self.content_mappers: # Use existing mappers for other content types mapped_events = await self.content_mappers[content.type](content, context) @@ -899,10 +931,14 @@ class MessageMapper: return events - # Handle output events separately to preserve output data - if event_type == "output": + # Handle yield events (output / intermediate / data) by extracting visible + # text from the payload. All three render as a visible message item so the + # gap that previously dropped intermediate yields into generic completed- + # trace events is closed. + if event_type in ("output", "intermediate", "data"): output_data = getattr(event, "data", None) executor_id = getattr(event, "executor_id", "unknown") + workflow_metadata = _workflow_output_metadata(event_type, executor_id) if output_data is not None: # Import required types @@ -960,6 +996,8 @@ class MessageMapper: content=[text_content], status="completed", ) + if workflow_metadata is not None: + cast(Any, output_message).metadata = workflow_metadata # Emit output_item.added for each yield_output logger.debug( diff --git a/python/packages/devui/frontend/src/components/features/workflow/execution-timeline.tsx b/python/packages/devui/frontend/src/components/features/workflow/execution-timeline.tsx index b9b2fc7da4..2dca6d8a59 100644 --- a/python/packages/devui/frontend/src/components/features/workflow/execution-timeline.tsx +++ b/python/packages/devui/frontend/src/components/features/workflow/execution-timeline.tsx @@ -96,6 +96,14 @@ function getStateBadgeClass(state: ExecutorState) { } } +function getMessageText(item: unknown): string { + const content = (item as { content?: Array<{ type: string; text?: string }> }).content; + return content + ?.filter((content) => content.type === "output_text" && content.text) + .map((content) => content.text) + .join("\n") ?? ""; +} + function ExecutorRunItem({ run, isExpanded, @@ -282,7 +290,12 @@ export function ExecutionTimeline({ }); } else if (item && item.type === "message" && "metadata" in item && item.id) { // Handle message items from Magentic agents - const metadata = item.metadata as { agent_id?: string; source?: string } | undefined; + const metadata = item.metadata as { + agent_id?: string; + executor_id?: string; + source?: string; + workflow_output_kind?: string; + } | undefined; if (metadata?.agent_id && metadata?.source === "magentic") { const executorId = metadata.agent_id; const itemId = item.id; @@ -298,6 +311,21 @@ export function ExecutionTimeline({ timestamp: uiTimestamp, runNumber, }); + } else if (metadata?.executor_id && metadata.workflow_output_kind === "intermediate") { + const executorId = metadata.executor_id; + const itemId = item.id; + const runNumber = (runCount.get(executorId) || 0) + 1; + runCount.set(executorId, runNumber); + + runs.push({ + executorId, + executorName: truncateText(executorId, 35), + itemId, + state: item.status === "completed" ? "completed" : "running", + output: itemOutputs[itemId] || getMessageText(item), + timestamp: uiTimestamp, + runNumber, + }); } } } @@ -327,7 +355,12 @@ export function ExecutionTimeline({ } } else if (item && item.type === "message" && "metadata" in item && item.id) { // Handle message completion from Magentic agents - const metadata = item.metadata as { agent_id?: string; source?: string } | undefined; + const metadata = item.metadata as { + agent_id?: string; + executor_id?: string; + source?: string; + workflow_output_kind?: string; + } | undefined; if (metadata?.agent_id && metadata?.source === "magentic") { const itemId = item.id; const existingRun = runs.find((r) => r.itemId === itemId); @@ -336,6 +369,14 @@ export function ExecutionTimeline({ existingRun.state = item.status === "completed" ? "completed" : "failed"; existingRun.output = itemOutputs[itemId] || ""; } + } else if (metadata?.executor_id && metadata.workflow_output_kind === "intermediate") { + const itemId = item.id; + const existingRun = runs.find((r) => r.itemId === itemId); + + if (existingRun) { + existingRun.state = item.status === "completed" ? "completed" : "failed"; + existingRun.output = itemOutputs[itemId] || getMessageText(item); + } } } } diff --git a/python/packages/devui/frontend/src/components/features/workflow/workflow-view.tsx b/python/packages/devui/frontend/src/components/features/workflow/workflow-view.tsx index 4696ef57a5..7edeefb8ba 100644 --- a/python/packages/devui/frontend/src/components/features/workflow/workflow-view.tsx +++ b/python/packages/devui/frontend/src/components/features/workflow/workflow-view.tsx @@ -663,6 +663,7 @@ export function WorkflowView({ item && item.type === "message" && (!("metadata" in item) || !(item.metadata as { source?: string } | undefined)?.source) && + (item.metadata as { workflow_output_kind?: string } | undefined)?.workflow_output_kind !== "intermediate" && "content" in item && Array.isArray(item.content) ) { @@ -1121,27 +1122,30 @@ export function WorkflowView({ // Handle workflow output messages if (item && item.type === "message" && "content" in item && Array.isArray(item.content)) { - // Extract text from message content - for (const content of item.content as Array<{ type: string; text?: string }>) { - if (content.type === "output_text" && content.text) { - const text = content.text; // Capture for closure - // Append to workflow result (support multiple yield_output calls) - setWorkflowResult((prev) => { - if (prev && prev.length > 0) { - // If there's existing output, add separator - return prev + "\n\n" + text; - } - return text; - }); + const metadata = item.metadata as { workflow_output_kind?: string } | undefined; + if (metadata?.workflow_output_kind !== "intermediate") { + // Extract text from message content + for (const content of item.content as Array<{ type: string; text?: string }>) { + if (content.type === "output_text" && content.text) { + const text = content.text; // Capture for closure + // Append to workflow result (support multiple yield_output calls) + setWorkflowResult((prev) => { + if (prev && prev.length > 0) { + // If there's existing output, add separator + return prev + "\n\n" + text; + } + return text; + }); - // Try to parse as JSON for structured metadata - try { - const parsed = JSON.parse(text); - if (typeof parsed === "object" && parsed !== null) { - workflowMetadata.current = parsed; + // Try to parse as JSON for structured metadata + try { + const parsed = JSON.parse(text); + if (typeof parsed === "object" && parsed !== null) { + workflowMetadata.current = parsed; + } + } catch { + // Not JSON, keep as text } - } catch { - // Not JSON, keep as text } } } diff --git a/python/packages/devui/frontend/src/types/openai.ts b/python/packages/devui/frontend/src/types/openai.ts index 31f7b20f02..9fba290027 100644 --- a/python/packages/devui/frontend/src/types/openai.ts +++ b/python/packages/devui/frontend/src/types/openai.ts @@ -376,6 +376,7 @@ export interface ResponseTextDeltaEvent extends ResponseStreamEvent { content_index: number; sequence_number: number; logprobs: Record[]; + metadata?: Record; } // OpenAI Response for non-streaming @@ -397,6 +398,7 @@ export interface ResponseOutputMessage { content: ResponseOutputText[]; id: string; status: "completed" | "failed" | "in_progress"; + metadata?: Record; } export interface ResponseOutputText { diff --git a/python/packages/devui/tests/devui/test_mapper.py b/python/packages/devui/tests/devui/test_mapper.py index b26900a89f..3ff4492d80 100644 --- a/python/packages/devui/tests/devui/test_mapper.py +++ b/python/packages/devui/tests/devui/test_mapper.py @@ -517,7 +517,8 @@ async def test_magentic_executor_event_with_agent_delta_metadata( """Test that WorkflowEvent[AgentResponseUpdate] with magentic_event_type='agent_delta' is handled correctly. This tests the ACTUAL event format Magentic emits - not a fake MagenticAgentDeltaEvent class. - Magentic uses WorkflowEvent.emit() with additional_properties containing magentic_event_type. + Magentic emits type='intermediate' WorkflowEvent instances with additional_properties + containing magentic_event_type. """ from agent_framework._types import AgentResponseUpdate from agent_framework._workflows._events import WorkflowEvent @@ -532,7 +533,7 @@ async def test_magentic_executor_event_with_agent_delta_metadata( "agent_id": "writer_agent", }, ) - event = WorkflowEvent.emit(executor_id="magentic_executor", data=update) + event = WorkflowEvent("intermediate", executor_id="magentic_executor", data=update) events = await mapper.convert_event(event, test_request) @@ -547,8 +548,8 @@ async def test_magentic_executor_event_with_agent_delta_metadata( async def test_magentic_orchestrator_message_event(mapper: MessageMapper, test_request: AgentFrameworkRequest) -> None: """Test that WorkflowEvent[AgentResponseUpdate] with magentic_event_type='orchestrator_message' is handled. - Magentic emits orchestrator planning/instruction messages using WorkflowEvent.emit() - with additional_properties containing magentic_event_type='orchestrator_message'. + Magentic emits orchestrator planning/instruction messages using type='intermediate' + WorkflowEvent instances with additional_properties containing magentic_event_type='orchestrator_message'. """ from agent_framework._types import AgentResponseUpdate from agent_framework._workflows._events import WorkflowEvent @@ -564,7 +565,7 @@ async def test_magentic_orchestrator_message_event(mapper: MessageMapper, test_r "orchestrator_id": "magentic_orchestrator", }, ) - event = WorkflowEvent.emit(executor_id="magentic_orchestrator", data=update) + event = WorkflowEvent("intermediate", executor_id="magentic_orchestrator", data=update) events = await mapper.convert_event(event, test_request) @@ -595,7 +596,7 @@ async def test_magentic_events_use_same_event_class_as_other_workflows( contents=[Content.from_text(text="Regular workflow response")], role="assistant", ) - regular_event = WorkflowEvent.emit(executor_id="regular_executor", data=regular_update) + regular_event = WorkflowEvent("intermediate", executor_id="regular_executor", data=regular_update) # 2. Magentic workflow (with additional_properties) magentic_update = AgentResponseUpdate( @@ -603,7 +604,7 @@ async def test_magentic_events_use_same_event_class_as_other_workflows( role="assistant", additional_properties={"magentic_event_type": "agent_delta"}, ) - magentic_event = WorkflowEvent.emit(executor_id="magentic_executor", data=magentic_update) + magentic_event = WorkflowEvent("intermediate", executor_id="magentic_executor", data=magentic_update) # Both should be the SAME class assert type(regular_event) is type(magentic_event) @@ -653,7 +654,7 @@ async def test_workflow_output_event(mapper: MessageMapper, test_request: AgentF """Test output event (type='output') is converted to output_item.added.""" from agent_framework._workflows._events import WorkflowEvent - event = WorkflowEvent.output(executor_id="final_executor", data="Final workflow output") + event = WorkflowEvent("output", executor_id="final_executor", data="Final workflow output") events = await mapper.convert_event(event, test_request) # output event (type='output') should emit output_item.added @@ -662,6 +663,9 @@ async def test_workflow_output_event(mapper: MessageMapper, test_request: AgentF # Check item contains the output text item = events[0].item assert item.type == "message" + assert item.metadata["workflow_event_type"] == "output" + assert item.metadata["workflow_output_kind"] == "terminal" + assert item.metadata["executor_id"] == "final_executor" assert any("Final workflow output" in str(c) for c in item.content) @@ -675,13 +679,104 @@ async def test_workflow_output_event_with_list_data(mapper: MessageMapper, test_ Message(role="user", contents=[Content.from_text(text="Hello")]), Message(role="assistant", contents=[Content.from_text(text="World")]), ] - event = WorkflowEvent.output(executor_id="complete", data=messages) + event = WorkflowEvent("output", executor_id="complete", data=messages) events = await mapper.convert_event(event, test_request) assert len(events) == 1 assert events[0].type == "response.output_item.added" +async def test_workflow_intermediate_event_with_agent_response_update_dispatched( + mapper: MessageMapper, test_request: AgentFrameworkRequest +) -> None: + """A WorkflowEvent with type='intermediate' wrapping an AgentResponseUpdate is mapped + just like type='output' / type='data' — to OpenAI text-delta events.""" + from agent_framework._workflows._events import WorkflowEvent + + update = AgentResponseUpdate( + contents=[Content.from_text(text="intermediate progress")], + role="assistant", + author_name="non-designated-agent", + ) + event = WorkflowEvent("intermediate", executor_id="non_designated", data=update) + events = await mapper.convert_event(event, test_request) + + assert len(events) >= 1 + added_events = [e for e in events if getattr(e, "type", "") == "response.output_item.added"] + assert added_events + item = added_events[0].item + assert item.metadata["workflow_event_type"] == "intermediate" + assert item.metadata["workflow_output_kind"] == "intermediate" + assert item.metadata["executor_id"] == "non_designated" + text_events = [e for e in events if getattr(e, "type", "") == "response.output_text.delta"] + assert len(text_events) >= 1 + assert text_events[0].metadata["workflow_event_type"] == "intermediate" + assert text_events[0].metadata["workflow_output_kind"] == "intermediate" + assert text_events[0].metadata["executor_id"] == "non_designated" + assert text_events[0].delta == "intermediate progress" + + +async def test_workflow_intermediate_event_with_string_payload_renders_visible_text( + mapper: MessageMapper, test_request: AgentFrameworkRequest +) -> None: + """A WorkflowEvent with type='intermediate' wrapping a plain string surfaces as a + visible output item — not a generic completed-trace event. Without this, executors + that ``await ctx.yield_output("plan: …")`` from non-designated nodes are silently + dropped in DevUI.""" + from agent_framework._workflows._events import WorkflowEvent + + event = WorkflowEvent("intermediate", executor_id="planner", data="plan: starting work") + events = await mapper.convert_event(event, test_request) + + assert len(events) == 1 + assert events[0].type == "response.output_item.added" + item = events[0].item + assert item.type == "message" + assert item.metadata["workflow_event_type"] == "intermediate" + assert item.metadata["workflow_output_kind"] == "intermediate" + assert item.metadata["executor_id"] == "planner" + assert any("plan: starting work" in str(c) for c in item.content) + + +async def test_workflow_intermediate_event_with_message_payload_renders_visible_text( + mapper: MessageMapper, test_request: AgentFrameworkRequest +) -> None: + """type='intermediate' wrapping a Message surfaces visibly — same path as type='output'.""" + from agent_framework import Message + from agent_framework._workflows._events import WorkflowEvent + + msg = Message(role="assistant", contents=[Content.from_text(text="research note")]) + event = WorkflowEvent("intermediate", executor_id="researcher", data=msg) + events = await mapper.convert_event(event, test_request) + + assert len(events) == 1 + assert events[0].type == "response.output_item.added" + item = events[0].item + assert item.metadata["workflow_event_type"] == "intermediate" + assert item.metadata["workflow_output_kind"] == "intermediate" + assert item.metadata["executor_id"] == "researcher" + assert any("research note" in str(c) for c in item.content) + + +async def test_workflow_data_event_keeps_intermediate_compatibility_metadata( + mapper: MessageMapper, test_request: AgentFrameworkRequest +) -> None: + """Deprecated type='data' workflow events remain visible and explicitly intermediate.""" + from agent_framework._workflows._events import WorkflowEvent + + with pytest.warns(DeprecationWarning): + event = WorkflowEvent.emit(executor_id="legacy", data="legacy progress") + events = await mapper.convert_event(event, test_request) + + assert len(events) == 1 + assert events[0].type == "response.output_item.added" + item = events[0].item + assert item.metadata["workflow_event_type"] == "data" + assert item.metadata["workflow_output_kind"] == "intermediate" + assert item.metadata["executor_id"] == "legacy" + assert any("legacy progress" in str(c) for c in item.content) + + # ============================================================================= # failed event (type='failed') Tests # ============================================================================= diff --git a/python/packages/foundry/tests/test_foundry_evals.py b/python/packages/foundry/tests/test_foundry_evals.py index d11999b76a..937a3cf524 100644 --- a/python/packages/foundry/tests/test_foundry_evals.py +++ b/python/packages/foundry/tests/test_foundry_evals.py @@ -1816,7 +1816,7 @@ class TestEvaluateWorkflow: WorkflowEvent.executor_completed("writer", [aer1]), WorkflowEvent.executor_invoked("reviewer", [aer1]), WorkflowEvent.executor_completed("reviewer", [aer2]), - WorkflowEvent.output("end", final_output), + WorkflowEvent("output", executor_id="end", data=final_output), ] wf_result = WorkflowRunResult(events, []) @@ -1845,7 +1845,7 @@ class TestEvaluateWorkflow: events = [ WorkflowEvent.executor_invoked("agent", "Test query"), WorkflowEvent.executor_completed("agent", [aer]), - WorkflowEvent.output("end", final_output), + WorkflowEvent("output", executor_id="end", data=final_output), ] wf_result = WorkflowRunResult(events, []) @@ -1875,7 +1875,7 @@ class TestEvaluateWorkflow: WorkflowEvent.executor_completed("input-conversation", None), WorkflowEvent.executor_invoked("planner", "Plan trip"), WorkflowEvent.executor_completed("planner", [aer]), - WorkflowEvent.output("end", final_output), + WorkflowEvent("output", executor_id="end", data=final_output), ] wf_result = WorkflowRunResult(events, []) @@ -1941,7 +1941,7 @@ class TestEvaluateWorkflow: WorkflowEvent.executor_completed("input-conversation", None), WorkflowEvent.executor_invoked("researcher", "What's the weather?"), WorkflowEvent.executor_completed("researcher", [aer]), - WorkflowEvent.output("end", [Message("assistant", ["Weather is sunny"])]), + WorkflowEvent("output", executor_id="end", data=[Message("assistant", ["Weather is sunny"])]), ] wf_result = WorkflowRunResult(events, []) @@ -2050,7 +2050,7 @@ class TestEvaluateWorkflow: events = [ WorkflowEvent.executor_invoked("agent", "Test query"), WorkflowEvent.executor_completed("agent", [aer]), - WorkflowEvent.output("end", final_output), + WorkflowEvent("output", executor_id="end", data=final_output), ] wf_result = WorkflowRunResult(events, []) @@ -2089,7 +2089,7 @@ class TestEvaluateWorkflow: events = [ WorkflowEvent.executor_invoked("agent", "Test query"), WorkflowEvent.executor_completed("agent", [aer]), - WorkflowEvent.output("end", final_output), + WorkflowEvent("output", executor_id="end", data=final_output), ] wf_result = WorkflowRunResult(events, []) diff --git a/python/packages/orchestrations/README.md b/python/packages/orchestrations/README.md index f965111712..63fd7ea4ee 100644 --- a/python/packages/orchestrations/README.md +++ b/python/packages/orchestrations/README.md @@ -18,6 +18,12 @@ Chain agents/executors in sequence, passing conversation context along: from agent_framework.orchestrations import SequentialBuilder workflow = SequentialBuilder(participants=[agent1, agent2, agent3]).build() + +# Preserve agent1 and agent2 as visible progress, while the default builder output remains Workflow Output. +workflow = SequentialBuilder( + participants=[agent1, agent2, agent3], + intermediate_output_from=[agent1, agent2], +).build() ``` ### ConcurrentBuilder @@ -55,6 +61,7 @@ from agent_framework.orchestrations import GroupChatBuilder workflow = GroupChatBuilder( participants=[agent1, agent2], selection_func=my_selector, + intermediate_output_from=[agent1, agent2], ).build() ``` @@ -68,9 +75,47 @@ from agent_framework.orchestrations import MagenticBuilder workflow = MagenticBuilder( participants=[researcher, writer, reviewer], manager_agent=manager_agent, + intermediate_output_from=[researcher, writer, reviewer], ).build() ``` +## Output Selection + +Orchestration builders expose Workflow Output selection using participant names. The core rule is that `output_from` +is an allow-list for Workflow Output, not a routing rule for every other participant output. Unselected participant +payloads are hidden unless `intermediate_output_from` explicitly selects them as Intermediate Output. + +- `output_from` designates participant emissions as Workflow Output (`type='output'` events). +- `intermediate_output_from` designates participant emissions as Intermediate Output (`type='intermediate'` events). + +If neither list is provided, each builder uses its documented default Workflow Output contract. Sequential emits the +last participant; Concurrent, GroupChat, and Magentic emit their aggregator/orchestrator/manager output; Handoff emits +participants. + +| Selection | Workflow Output | Intermediate Output | Hidden payloads | +| --- | --- | --- | --- | +| Omit both selections | Builder default Workflow Output contract | None | Builder-specific non-output participant payloads | +| `output_from="all"` | Every output-capable participant | None | None | +| `output_from=[writer]` | Only `writer` | None | All other participant payloads | +| `output_from=[writer], intermediate_output_from="all_other"` | Only `writer` | Every output-capable participant not selected by `output_from` | None | +| `intermediate_output_from="all_other"` | None, except builder-internal default output executors where applicable | Every output-capable participant | Builder-internal plumbing payloads | +| `output_from=[], intermediate_output_from="all_other"` | None, except builder-internal default output executors where applicable | Every output-capable participant | Builder-internal plumbing payloads | +| `output_from=[writer], intermediate_output_from=[researcher, reviewer]` | Only `writer` | `researcher` and `reviewer` | Any other participant payloads | + +Invalid selections fail at construction or build time: + +| Invalid selection | Why it fails | +| --- | --- | +| `output_from="all_other"` | `"all_other"` is only valid for `intermediate_output_from` | +| `intermediate_output_from="all"` | `"all"` is only valid for `output_from` | +| The same participant in both selections | One payload cannot be both Workflow Output and Intermediate Output | +| Duplicate participant selections | Duplicates are treated as configuration errors | +| Unknown participant selections | Typos and missing participants are rejected | +| `output_from=[], intermediate_output_from=[]` | Both explicit selections are empty | + +When an orchestration is wrapped with `workflow.as_agent()`, Workflow Output becomes normal response text. Intermediate +Output becomes `text_reasoning` content so callers can inspect progress without changing `.text` behavior. + ## Documentation For more information, see the [Agent Framework documentation](https://aka.ms/agent-framework). diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_concurrent.py b/python/packages/orchestrations/agent_framework_orchestrations/_concurrent.py index e1a931019a..6fc29c79b3 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_concurrent.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_concurrent.py @@ -4,7 +4,7 @@ import asyncio import inspect import logging from collections.abc import Callable, Sequence -from typing import Any +from typing import Any, Literal, cast from agent_framework import AgentResponse, Message, SupportsAgentRun from agent_framework._workflows._agent_executor import AgentExecutor, AgentExecutorRequest, AgentExecutorResponse @@ -18,6 +18,14 @@ from agent_framework._workflows._workflow_context import WorkflowContext from typing_extensions import Never from ._orchestration_request_info import AgentApprovalExecutor +from ._participant_output_config import ( + _MISSING, # pyright: ignore[reportPrivateUsage] + _coalesce_output_from, # pyright: ignore[reportPrivateUsage] + _coerce_intermediate_output_from, # pyright: ignore[reportPrivateUsage] + _ParticipantIntermediateOutputSelection, # pyright: ignore[reportPrivateUsage] + _ParticipantOutputSpecifier, # pyright: ignore[reportPrivateUsage] + _resolve_participant_output_config, # pyright: ignore[reportPrivateUsage] +) logger = logging.getLogger(__name__) @@ -205,23 +213,28 @@ class ConcurrentBuilder: *, participants: Sequence[SupportsAgentRun | Executor], checkpoint_storage: CheckpointStorage | None = None, - intermediate_outputs: bool = False, + output_from: Sequence[_ParticipantOutputSpecifier] | Literal["all"] | None = cast(Any, _MISSING), + intermediate_output_from: _ParticipantIntermediateOutputSelection = None, ) -> None: """Initialize the ConcurrentBuilder. Args: participants: Sequence of agent or executor instances to run in parallel. checkpoint_storage: Optional checkpoint storage for enabling workflow state persistence. - intermediate_outputs: If True, every participant's `yield_output` surfaces as a - workflow `output` event in addition to the aggregator's. By default - (False) only the aggregator's output surfaces. + output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``output`` events alongside the aggregator. Pass ``"all"`` to select every + participant. + intermediate_output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``intermediate`` events. Pass ``"all_other"`` to select every participant + not selected by ``output_from``. Unlisted participant outputs are hidden. """ self._participants: list[SupportsAgentRun | Executor] = [] self._aggregator: Executor | None = None self._checkpoint_storage: CheckpointStorage | None = checkpoint_storage self._request_info_enabled: bool = False self._request_info_filter: set[str] | None = None - self._intermediate_outputs: bool = intermediate_outputs + self._output_from = _coalesce_output_from(output_from=output_from) + self._intermediate_output_from = _coerce_intermediate_output_from(intermediate_output_from) self._set_participants(participants) @@ -396,10 +409,19 @@ class ConcurrentBuilder: # Resolve participants and participant factories to executors participants: list[Executor] = self._resolve_participants() + # Default: only the aggregator is terminal; participant outputs are hidden + # unless explicitly designated as terminal or intermediate. + designated, intermediate_designated = _resolve_participant_output_config( + participants=participants, + output_from=self._output_from, + intermediate_output_from=self._intermediate_output_from, + extra_output_executors=[aggregator], + ) builder = WorkflowBuilder( start_executor=dispatcher, checkpoint_storage=self._checkpoint_storage, - output_executors=[aggregator] if not self._intermediate_outputs else None, + output_from=designated, + intermediate_output_from=intermediate_designated, ) # Fan-out for parallel execution builder.add_fan_out_edges(dispatcher, participants) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_group_chat.py b/python/packages/orchestrations/agent_framework_orchestrations/_group_chat.py index 9f7e011252..3778e5d110 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_group_chat.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_group_chat.py @@ -27,7 +27,7 @@ import sys from collections import OrderedDict from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass -from typing import Any, ClassVar, cast +from typing import Any, ClassVar, Literal, cast from agent_framework import Agent, AgentResponse, AgentResponseUpdate, AgentSession, Message, SupportsAgentRun from agent_framework._workflows._agent_executor import AgentExecutor, AgentExecutorRequest, AgentExecutorResponse @@ -51,6 +51,14 @@ from ._base_group_chat_orchestrator import ( ) from ._orchestration_request_info import AgentApprovalExecutor from ._orchestrator_helpers import clean_conversation_for_handoff +from ._participant_output_config import ( + _MISSING, # pyright: ignore[reportPrivateUsage] + _coalesce_output_from, # pyright: ignore[reportPrivateUsage] + _coerce_intermediate_output_from, # pyright: ignore[reportPrivateUsage] + _ParticipantIntermediateOutputSelection, # pyright: ignore[reportPrivateUsage] + _ParticipantOutputSpecifier, # pyright: ignore[reportPrivateUsage] + _resolve_participant_output_config, # pyright: ignore[reportPrivateUsage] +) if sys.version_info >= (3, 12): from typing import override # type: ignore # pragma: no cover @@ -618,7 +626,8 @@ class GroupChatBuilder: termination_condition: TerminationCondition | None = None, max_rounds: int | None = None, checkpoint_storage: CheckpointStorage | None = None, - intermediate_outputs: bool = False, + output_from: Sequence[_ParticipantOutputSpecifier] | Literal["all"] | None = cast(Any, _MISSING), + intermediate_output_from: _ParticipantIntermediateOutputSelection = None, ) -> None: """Initialize the GroupChatBuilder. @@ -635,9 +644,12 @@ class GroupChatBuilder: True to terminate the conversation, False to continue. max_rounds: Optional maximum number of orchestrator rounds to prevent infinite conversations. checkpoint_storage: Optional checkpoint storage for enabling workflow state persistence. - intermediate_outputs: If True, every participant's `yield_output` surfaces as a - workflow `output` event in addition to the orchestrator's. By default (False) - only the orchestrator's output surfaces. + output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``output`` events alongside the orchestrator. Pass ``"all"`` to select every + participant. + intermediate_output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``intermediate`` events. Pass ``"all_other"`` to select every participant + not selected by ``output_from``. Unlisted participant outputs are hidden. """ self._participants: dict[str, SupportsAgentRun | Executor] = {} self._participant_factories: list[Callable[[], SupportsAgentRun | Executor]] = [] @@ -658,7 +670,8 @@ class GroupChatBuilder: self._request_info_enabled: bool = False self._request_info_filter: set[str] = set() - self._intermediate_outputs: bool = intermediate_outputs + self._output_from = _coalesce_output_from(output_from=output_from) + self._intermediate_output_from = _coerce_intermediate_output_from(intermediate_output_from) if participants is None and participant_factories is None: raise ValueError("Either participants or participant_factories must be provided.") @@ -1001,11 +1014,20 @@ class GroupChatBuilder: participants: list[Executor] = self._resolve_participants() orchestrator: Executor = self._resolve_orchestrator(participants) - # Build workflow graph + # Default: only the orchestrator is terminal; participant outputs are hidden + # unless explicitly designated as terminal or intermediate. + # `group_chat` orchestrator-progress events keep their dedicated event type. + designated, intermediate_designated = _resolve_participant_output_config( + participants=participants, + output_from=self._output_from, + intermediate_output_from=self._intermediate_output_from, + extra_output_executors=[orchestrator], + ) workflow_builder = WorkflowBuilder( start_executor=orchestrator, checkpoint_storage=self._checkpoint_storage, - output_executors=[orchestrator] if not self._intermediate_outputs else None, + output_from=designated, + intermediate_output_from=intermediate_designated, ) for participant in participants: # Orchestrator and participant bi-directional edges diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_handoff.py b/python/packages/orchestrations/agent_framework_orchestrations/_handoff.py index f555ab89b0..70f28e7f04 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_handoff.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_handoff.py @@ -36,7 +36,7 @@ import sys from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass -from typing import Any +from typing import Any, Literal, cast from agent_framework import Agent, AgentResponse, Message, SupportsAgentRun from agent_framework._middleware import FunctionInvocationContext, FunctionMiddleware, MiddlewareTermination @@ -53,6 +53,14 @@ from agent_framework._workflows._workflow_context import WorkflowContext from ._base_group_chat_orchestrator import TerminationCondition from ._orchestrator_helpers import clean_conversation_for_handoff +from ._participant_output_config import ( + _MISSING, # pyright: ignore[reportPrivateUsage] + _coalesce_output_from, # pyright: ignore[reportPrivateUsage] + _coerce_intermediate_output_from, # pyright: ignore[reportPrivateUsage] + _ParticipantIntermediateOutputSelection, # pyright: ignore[reportPrivateUsage] + _ParticipantOutputSpecifier, # pyright: ignore[reportPrivateUsage] + _resolve_participant_output_config, # pyright: ignore[reportPrivateUsage] +) if sys.version_info >= (3, 12): from typing import override # type: ignore # pragma: no cover @@ -377,7 +385,7 @@ class HandoffAgentExecutor(AgentExecutor): # Append the agent response to the full conversation history. This list removes # function call related content such that the result stays consistent regardless - # of which agent yields the final output. + # of which agent yields Workflow Output. self._full_conversation.extend(cleaned_response) # Broadcast only the cleaned response to other agents (without function_calls/results) @@ -577,7 +585,7 @@ class HandoffBuilder: Note: 1. Agents in handoff workflows must be ``Agent`` instances and support local tool calls. - 2. Because each agent's response is itself a workflow output, handoff has no separate + 2. Because each agent's response is itself Workflow Output, handoff has no separate "intermediate outputs" channel — every per-agent response is the primary output. """ @@ -589,6 +597,8 @@ class HandoffBuilder: description: str | None = None, checkpoint_storage: CheckpointStorage | None = None, termination_condition: TerminationCondition | None = None, + output_from: Sequence[_ParticipantOutputSpecifier] | Literal["all"] | None = cast(Any, _MISSING), + intermediate_output_from: _ParticipantIntermediateOutputSelection = None, ) -> None: r"""Initialize a HandoffBuilder for creating conversational handoff workflows. @@ -610,6 +620,12 @@ class HandoffBuilder: checkpoint_storage: Optional checkpoint storage for enabling workflow state persistence. termination_condition: Optional callable that receives the full conversation and returns True (or awaitable True) if the workflow should terminate. + output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``output`` events. Defaults to all participants; pass ``"all"`` to select every + participant explicitly. + intermediate_output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``intermediate`` events. Pass ``"all_other"`` to select every participant + not selected by ``output_from``. Unlisted participant outputs are hidden. """ self._name = name self._description = description @@ -635,6 +651,8 @@ class HandoffBuilder: # Termination related members self._termination_condition: Callable[[list[Message]], bool | Awaitable[bool]] | None = termination_condition + self._output_from = _coalesce_output_from(output_from=output_from) + self._intermediate_output_from = _coerce_intermediate_output_from(intermediate_output_from) def participants(self, participants: Sequence[Agent]) -> "HandoffBuilder": """Register the agents that will participate in the handoff workflow. @@ -955,11 +973,22 @@ class HandoffBuilder: if self._start_id is None: raise ValueError("Must call with_start_agent(...) before building the workflow.") start_executor = executors[self._resolve_to_id(resolved_agents[self._start_id])] + # Handoff has no separate terminator: every participant's reply is a primary + # output by default. Explicit participant designation can narrow or reclassify + # selected speakers. + output, intermediate_output = _resolve_participant_output_config( + participants=list(executors.values()), + output_from=self._output_from, + intermediate_output_from=self._intermediate_output_from, + default_output_from=list(executors.values()), + ) builder = WorkflowBuilder( name=self._name, description=self._description, start_executor=start_executor, checkpoint_storage=self._checkpoint_storage, + output_from=output, + intermediate_output_from=intermediate_output, ) # Add the appropriate edges diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py index 7f1854f914..53ca4052ff 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py @@ -10,7 +10,7 @@ from abc import ABC, abstractmethod from collections.abc import Callable, Sequence from dataclasses import dataclass, field from enum import Enum -from typing import Any, ClassVar, TypeVar, cast +from typing import Any, ClassVar, Literal, TypeVar, cast from agent_framework import ( AgentResponse, @@ -38,6 +38,14 @@ from ._base_group_chat_orchestrator import ( GroupChatWorkflowContextOutT, ParticipantRegistry, ) +from ._participant_output_config import ( + _MISSING, # pyright: ignore[reportPrivateUsage] + _coalesce_output_from, # pyright: ignore[reportPrivateUsage] + _coerce_intermediate_output_from, # pyright: ignore[reportPrivateUsage] + _ParticipantIntermediateOutputSelection, # pyright: ignore[reportPrivateUsage] + _ParticipantOutputSpecifier, # pyright: ignore[reportPrivateUsage] + _resolve_participant_output_config, # pyright: ignore[reportPrivateUsage] +) if sys.version_info >= (3, 12): from typing import override # type: ignore # pragma: no cover @@ -1409,7 +1417,8 @@ class MagenticBuilder: # Existing params enable_plan_review: bool = False, checkpoint_storage: CheckpointStorage | None = None, - intermediate_outputs: bool = False, + output_from: Sequence[_ParticipantOutputSpecifier] | Literal["all"] | None = cast(Any, _MISSING), + intermediate_output_from: _ParticipantIntermediateOutputSelection = None, ) -> None: """Initialize the Magentic workflow builder. @@ -1432,9 +1441,12 @@ class MagenticBuilder: max_round_count: Max total coordination rounds. None means unlimited. enable_plan_review: If True, requires human approval of the initial plan before proceeding. checkpoint_storage: Optional checkpoint storage for enabling workflow state persistence. - intermediate_outputs: If True, every participant's `yield_output` surfaces as a - workflow `output` event in addition to the orchestrator's. By default (False) - only the orchestrator's output surfaces. + output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``output`` events alongside the manager. Pass ``"all"`` to select every + participant. + intermediate_output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``intermediate`` events. Pass ``"all_other"`` to select every participant + not selected by ``output_from``. Unlisted participant outputs are hidden. """ self._participants: dict[str, SupportsAgentRun | Executor] = {} @@ -1447,7 +1459,8 @@ class MagenticBuilder: self._checkpoint_storage: CheckpointStorage | None = checkpoint_storage - self._intermediate_outputs = intermediate_outputs + self._output_from = _coalesce_output_from(output_from=output_from) + self._intermediate_output_from = _coerce_intermediate_output_from(intermediate_output_from) self._set_participants(participants) @@ -1762,11 +1775,20 @@ class MagenticBuilder: participants: list[Executor] = self._resolve_participants() orchestrator: Executor = self._resolve_orchestrator(participants) - # Build workflow graph + # Default: only the manager is terminal; worker outputs are hidden unless + # explicitly designated as terminal or intermediate. + # `magentic_orchestrator` events keep their dedicated event type. + designated, intermediate_designated = _resolve_participant_output_config( + participants=participants, + output_from=self._output_from, + intermediate_output_from=self._intermediate_output_from, + extra_output_executors=[orchestrator], + ) workflow_builder = WorkflowBuilder( start_executor=orchestrator, checkpoint_storage=self._checkpoint_storage, - output_executors=[orchestrator] if not self._intermediate_outputs else None, + output_from=designated, + intermediate_output_from=intermediate_designated, ) for participant in participants: # Orchestrator and participant bi-directional edges diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_orchestration_request_info.py b/python/packages/orchestrations/agent_framework_orchestrations/_orchestration_request_info.py index 23e382aa13..66949ae576 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_orchestration_request_info.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_orchestration_request_info.py @@ -220,8 +220,14 @@ class AgentApprovalExecutor(WorkflowExecutor): request_info_cls = _TerminalAgentRequestInfoExecutor if terminal else AgentRequestInfoExecutor request_info_executor = request_info_cls(id="agent_request_info_executor") + # Both inner executors yield the inner workflow's terminal output (the agent + # during its turn; the _TerminalAgentRequestInfoExecutor after approval), so + # both must be designated for WorkflowExecutor.get_outputs() to surface them. return ( - WorkflowBuilder(start_executor=agent_executor) + WorkflowBuilder( + start_executor=agent_executor, + output_from=[agent_executor, request_info_executor], + ) # Create a loop between agent executor and request info executor .add_edge(agent_executor, request_info_executor) .add_edge(request_info_executor, agent_executor) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_participant_output_config.py b/python/packages/orchestrations/agent_framework_orchestrations/_participant_output_config.py new file mode 100644 index 0000000000..49138b7d0d --- /dev/null +++ b/python/packages/orchestrations/agent_framework_orchestrations/_participant_output_config.py @@ -0,0 +1,166 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Participant-oriented workflow output configuration helpers.""" + +from collections.abc import Sequence +from typing import Any, Literal + +from agent_framework import SupportsAgentRun +from agent_framework._workflows._agent_utils import resolve_agent_id +from agent_framework._workflows._executor import Executor + +_MISSING = object() +_ALL_OUTPUTS: Literal["all"] = "all" +_ALL_OTHER_OUTPUTS: Literal["all_other"] = "all_other" +_ParticipantOutputSpecifier = str | SupportsAgentRun | Executor +_ParticipantOutputSelection = Sequence[_ParticipantOutputSpecifier] | Literal["all"] | None +_ParticipantIntermediateOutputSelection = Sequence[_ParticipantOutputSpecifier] | Literal["all", "all_other"] | None +_WorkflowExecutorSpecifier = Executor | SupportsAgentRun + + +def _coalesce_output_from( # pyright: ignore[reportUnusedFunction] + *, + output_from: Any = _MISSING, +) -> _ParticipantOutputSelection: + """Resolve orchestration output selection to ``output_from``.""" + if output_from is not _MISSING: + return _coerce_output_from(output_from) + return None + + +def _coerce_output_from(output_from: Any) -> _ParticipantOutputSelection: + """Coerce workflow-output participant selection while preserving the ``"all"`` literal.""" + if output_from is None: + return None + if isinstance(output_from, str): + if output_from == _ALL_OUTPUTS: + return _ALL_OUTPUTS + if output_from == _ALL_OTHER_OUTPUTS: + raise ValueError("output_from='all_other' is invalid; use intermediate_output_from='all_other' instead.") + raise ValueError(f"Unsupported output_from literal {output_from!r}; use 'all' or a list of participants.") + return list(output_from) + + +def _coerce_intermediate_output_from( # pyright: ignore[reportUnusedFunction] + intermediate_output_from: Any, +) -> _ParticipantIntermediateOutputSelection: + """Coerce intermediate-output participant selection while preserving ``"all_other"``.""" + if intermediate_output_from is None: + return None + if isinstance(intermediate_output_from, str): + if intermediate_output_from == _ALL_OUTPUTS: + return _ALL_OUTPUTS + if intermediate_output_from == _ALL_OTHER_OUTPUTS: + return _ALL_OTHER_OUTPUTS + raise ValueError( + f"Unsupported intermediate_output_from literal {intermediate_output_from!r}; " + "use 'all', 'all_other', or a list of participants." + ) + return list(intermediate_output_from) + + +def _resolve_participant_output_config( # pyright: ignore[reportUnusedFunction] + *, + participants: Sequence[Executor], + output_from: _ParticipantOutputSelection, + intermediate_output_from: _ParticipantIntermediateOutputSelection, + default_output_from: Sequence[Executor] = (), + extra_output_executors: Sequence[Executor] = (), +) -> tuple[list[_WorkflowExecutorSpecifier], list[_WorkflowExecutorSpecifier]]: + """Resolve public participant output config into workflow executor config.""" + explicit_config = output_from is not None or intermediate_output_from is not None + if explicit_config and not (output_from or intermediate_output_from): + raise ValueError("output_from and intermediate_output_from cannot both be empty.") + + participants_by_id = {participant.id: participant for participant in participants} + known_participants = sorted(participants_by_id) + + if output_from == _ALL_OUTPUTS: + output_designated = list(participants) + elif output_from is not None: + output_designated = _resolve_designated_participants( + output_from, + kind="output", + participants_by_id=participants_by_id, + known_participants=known_participants, + ) + elif intermediate_output_from in (_ALL_OTHER_OUTPUTS, _ALL_OUTPUTS): + output_designated = [] + else: + intermediate_designated = ( + _resolve_designated_participants( + intermediate_output_from, + kind="intermediate", + participants_by_id=participants_by_id, + known_participants=known_participants, + ) + if intermediate_output_from is not None + else [] + ) + # The caller-supplied default applies only to participants not explicitly designated as + # intermediate. Without this subtraction, builders that pre-populate a default output list + # (Handoff defaults to all participants, Sequential defaults to the last) would force + # an overlap error whenever a user passed `intermediate_output_from=[X]` for an X in + # the default set, contradicting the public docstring contract. + intermediate_ids = {participant.id for participant in intermediate_designated} + output_designated = [ + participant for participant in default_output_from if participant.id not in intermediate_ids + ] + + if intermediate_output_from == _ALL_OUTPUTS: + intermediate_designated = list(participants) + elif intermediate_output_from == _ALL_OTHER_OUTPUTS: + output_ids = {participant.id for participant in output_designated} + intermediate_designated = [participant for participant in participants if participant.id not in output_ids] + elif intermediate_output_from is not None: + intermediate_designated = _resolve_designated_participants( + intermediate_output_from, + kind="intermediate", + participants_by_id=participants_by_id, + known_participants=known_participants, + ) + else: + intermediate_designated = [] + + overlap = sorted( + {participant.id for participant in output_designated}.intersection( + participant.id for participant in intermediate_designated + ) + ) + if overlap: + raise ValueError(f"Participants cannot be both output and intermediate designated: {overlap}") + + output_executors: list[_WorkflowExecutorSpecifier] = [*extra_output_executors, *output_designated] + intermediate_executors: list[_WorkflowExecutorSpecifier] = list(intermediate_designated) + return output_executors, intermediate_executors + + +def _resolve_designated_participants( + designations: Sequence[_ParticipantOutputSpecifier], + *, + kind: str, + participants_by_id: dict[str, Executor], + known_participants: Sequence[str], +) -> list[Executor]: + resolved: list[Executor] = [] + seen: set[str] = set() + for designation in designations: + participant_id = _participant_id(designation) + if participant_id in seen: + raise ValueError(f"Duplicate {kind} participant '{participant_id}' in {kind}_participants.") + seen.add(participant_id) + try: + resolved.append(participants_by_id[participant_id]) + except KeyError as exc: + raise ValueError( + f"Unknown {kind} participant '{participant_id}'. Known participants: {known_participants}" + ) from exc + return resolved + + +def _participant_id(participant: _ParticipantOutputSpecifier) -> str: + if isinstance(participant, str): + return participant + if isinstance(participant, Executor): + return participant.id + return resolve_agent_id(participant) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_sequential.py b/python/packages/orchestrations/agent_framework_orchestrations/_sequential.py index 36d4f23f49..70796d5e26 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_sequential.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_sequential.py @@ -16,7 +16,7 @@ produces — by convention an `AgentResponse` so downstream consumers see a unif import logging from collections.abc import Sequence -from typing import Literal +from typing import Any, Literal, cast from agent_framework import Message, SupportsAgentRun from agent_framework._workflows._agent_executor import AgentExecutor @@ -32,6 +32,14 @@ from agent_framework._workflows._workflow_builder import WorkflowBuilder from agent_framework._workflows._workflow_context import WorkflowContext from ._orchestration_request_info import AgentApprovalExecutor +from ._participant_output_config import ( + _MISSING, # pyright: ignore[reportPrivateUsage] + _coalesce_output_from, # pyright: ignore[reportPrivateUsage] + _coerce_intermediate_output_from, # pyright: ignore[reportPrivateUsage] + _ParticipantIntermediateOutputSelection, # pyright: ignore[reportPrivateUsage] + _ParticipantOutputSpecifier, # pyright: ignore[reportPrivateUsage] + _resolve_participant_output_config, # pyright: ignore[reportPrivateUsage] +) logger = logging.getLogger(__name__) @@ -60,7 +68,7 @@ class SequentialBuilder: - The workflow wires participants in order, passing a list[Message] down the chain - Agents append their assistant messages to the conversation - Custom executors can transform/summarize and return a list[Message] - - The final output is the conversation produced by the last participant + - The default Workflow Output is the conversation produced by the last participant Usage: @@ -91,7 +99,8 @@ class SequentialBuilder: participants: Sequence[SupportsAgentRun | Executor], checkpoint_storage: CheckpointStorage | None = None, chain_only_agent_responses: bool = False, - intermediate_outputs: bool = False, + output_from: Sequence[_ParticipantOutputSpecifier] | Literal["all"] | None = cast(Any, _MISSING), + intermediate_output_from: _ParticipantIntermediateOutputSelection = None, ) -> None: """Initialize the SequentialBuilder. @@ -101,16 +110,19 @@ class SequentialBuilder: chain_only_agent_responses: If True, only agent responses are chained between agents. By default, the full conversation context is passed to the next agent. This also applies to Executor -> Agent transitions if the executor sends `AgentExecutorResponse`. - intermediate_outputs: If True, every participant's `yield_output` surfaces as a - workflow `output` event in addition to the terminator's. By default (False) only - the last participant's output surfaces. + output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``output`` events. Pass ``"all"`` to select every participant. + intermediate_output_from: Optional participant names or instances whose ``yield_output`` calls + surface as workflow ``intermediate`` events. Pass ``"all_other"`` to select every participant + not selected by ``output_from``. Unlisted participant outputs are hidden. """ self._participants: list[SupportsAgentRun | Executor] = [] self._checkpoint_storage: CheckpointStorage | None = checkpoint_storage self._chain_only_agent_responses: bool = chain_only_agent_responses self._request_info_enabled: bool = False self._request_info_filter: set[str] | None = None - self._intermediate_outputs: bool = intermediate_outputs + self._output_from = _coalesce_output_from(output_from=output_from) + self._intermediate_output_from = _coerce_intermediate_output_from(intermediate_output_from) self._set_participants(participants) @@ -225,8 +237,8 @@ class SequentialBuilder: - Custom `Executor`: receives `list[Message]` and forwards `list[Message]`. If used as the terminator, it must call `ctx.yield_output(AgentResponse(...))` instead of `ctx.send_message(...)` — its yield becomes the workflow's output. - - The last participant is registered as the workflow's `output_executor`, so the - terminator's own `yield_output` is the workflow's terminal output (`AgentResponse`, + - The last participant is selected as Workflow Output by default, so the + terminator's own `yield_output` is Workflow Output (`AgentResponse`, or per-chunk `AgentResponseUpdate` when streaming). """ input_conv = _InputToConversation(id="input-conversation") @@ -234,10 +246,19 @@ class SequentialBuilder: # Resolve participants and participant factories to executors participants: list[Executor] = self._resolve_participants() + # Default: only the terminator is terminal. Explicit participant designation + # can surface selected earlier participant outputs as terminal or intermediate. + designated, intermediate_designated = _resolve_participant_output_config( + participants=participants, + output_from=self._output_from, + intermediate_output_from=self._intermediate_output_from, + default_output_from=[participants[-1]], + ) builder = WorkflowBuilder( start_executor=input_conv, checkpoint_storage=self._checkpoint_storage, - output_executors=[participants[-1]] if not self._intermediate_outputs else None, + output_from=designated, + intermediate_output_from=intermediate_designated, ) prior: Executor | SupportsAgentRun = input_conv diff --git a/python/packages/orchestrations/tests/test_magentic.py b/python/packages/orchestrations/tests/test_magentic.py index 0389fad94e..5c94d2fb14 100644 --- a/python/packages/orchestrations/tests/test_magentic.py +++ b/python/packages/orchestrations/tests/test_magentic.py @@ -630,9 +630,14 @@ class StubAssistantsAgent(BaseAgent): async def _collect_agent_responses_setup(participant: SupportsAgentRun) -> list[Message]: captured: list[Message] = [] - wf = MagenticBuilder(participants=[participant], intermediate_outputs=True, manager=InvokeOnceManager()).build() + wf = MagenticBuilder( + participants=[participant], + output_from=[participant], + manager=InvokeOnceManager(), + ).build() - # Run a bounded stream to allow one invoke and then completion + # With output_from, participants are designated as outputs alongside + # the manager — so their streaming chunks surface as type='output' (not intermediate). events: list[WorkflowEvent] = [] async for ev in wf.run("task", stream=True): events.append(ev) diff --git a/python/packages/orchestrations/tests/test_orchestration_intermediate_vs_terminal.py b/python/packages/orchestrations/tests/test_orchestration_intermediate_vs_terminal.py new file mode 100644 index 0000000000..78086d9d41 --- /dev/null +++ b/python/packages/orchestrations/tests/test_orchestration_intermediate_vs_terminal.py @@ -0,0 +1,749 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Tests for orchestration intermediate vs terminal output labeling. + +Verifies that under the strict-output model: + - Sequential / Concurrent / GroupChat / Magentic designate their terminator, + aggregator, orchestrator, or manager as the sole output executor; per-step + yields from non-designated executors emit `type='intermediate'` events. + - Handoff designates ALL participants — every reply is `type='output'`. + - When wrapped via `workflow.as_agent()`, caller-facing workflow events surface + with their original content types. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterable, Awaitable, Callable +from typing import Any, ClassVar, Literal, overload + +import pytest +from agent_framework import ( + AgentResponse, + AgentResponseUpdate, + AgentRunInputs, + AgentSession, + BaseAgent, + Content, + Message, + ResponseStream, +) +from agent_framework.orchestrations import ( + ConcurrentBuilder, + GroupChatBuilder, + GroupChatState, + HandoffBuilder, + MagenticBuilder, + MagenticContext, + MagenticManagerBase, + MagenticProgressLedger, + MagenticProgressLedgerItem, + SequentialBuilder, +) + + +class _EchoAgent(BaseAgent): + """Minimal non-streaming agent that returns a single assistant message.""" + + @overload + def run( + self, + messages: AgentRunInputs | None = ..., + *, + stream: Literal[False] = ..., + session: AgentSession | None = ..., + **kwargs: Any, + ) -> Awaitable[AgentResponse[Any]]: ... + @overload + def run( + self, + messages: AgentRunInputs | None = ..., + *, + stream: Literal[True], + session: AgentSession | None = ..., + **kwargs: Any, + ) -> ResponseStream[AgentResponseUpdate, AgentResponse[Any]]: ... + + def run( + self, + messages: AgentRunInputs | None = None, + *, + stream: bool = False, + session: AgentSession | None = None, + **kwargs: Any, + ) -> Awaitable[AgentResponse[Any]] | ResponseStream[AgentResponseUpdate, AgentResponse[Any]]: + if stream: + + async def _stream() -> AsyncIterable[AgentResponseUpdate]: + yield AgentResponseUpdate( + contents=[Content.from_text(text=f"{self.name} reply")], author_name=self.name + ) + + return ResponseStream(_stream(), finalizer=AgentResponse.from_updates) + + async def _run() -> AgentResponse: + return AgentResponse(messages=[Message("assistant", [f"{self.name} reply"], author_name=self.name)]) + + return _run() + + +# --------------------------------------------------------------------------- +# Sequential +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_sequential_default_only_terminator_is_output() -> None: + """Default Sequential designates only the terminator; earlier participants are hidden.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder(participants=[a, b, c]).build() + + output_events: list[Any] = [] + intermediate_events: list[Any] = [] + async for event in workflow.run("hello", stream=True): + if event.type == "output": + output_events.append(event) + elif event.type == "intermediate": + intermediate_events.append(event) + + # Only the terminator (C) emits type='output'. + assert len(output_events) == 1 + assert "C" in {ev.executor_id for ev in output_events} + + assert not intermediate_events + + +@pytest.mark.asyncio +async def test_sequential_output_from_designates_workflow_output_participants() -> None: + """Sequential output_from controls which participant yields surface as workflow output.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder(participants=[a, b, c], output_from=["A", "B", "C"]).build() + result = await workflow.run("hello") + outputs = result.get_outputs() + assert len(outputs) == 3 + + +@pytest.mark.asyncio +async def test_sequential_intermediate_output_from_surface_as_intermediate() -> None: + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder(participants=[a, b, c], intermediate_output_from=[a, "B"]).build() + + output_executors: set[str] = set() + intermediate_executors: set[str] = set() + async for event in workflow.run("hello", stream=True): + if event.type == "output" and event.executor_id is not None: + output_executors.add(event.executor_id) + elif event.type == "intermediate" and event.executor_id is not None: + intermediate_executors.add(event.executor_id) + + assert output_executors == {"C"} + assert intermediate_executors == {"A", "B"} + + +@pytest.mark.asyncio +async def test_sequential_intermediate_can_demote_default_terminator() -> None: + """Regression: marking the default output terminator as intermediate must not raise an overlap error. + + Sequential's default output list is `[participants[-1]]`. Before the fix, designating that + same participant via `intermediate_output_from` triggered the + "Participants cannot be both output and intermediate designated" overlap rejection in + `_participant_output_config`, contradicting the public contract that + `intermediate_output_from` can be used independently of `output_from`. + """ + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder(participants=[a, b, c], intermediate_output_from=["C"]).build() + + output_executors: set[str] = set() + intermediate_executors: set[str] = set() + async for event in workflow.run("hello", stream=True): + if event.type == "output" and event.executor_id is not None: + output_executors.add(event.executor_id) + elif event.type == "intermediate" and event.executor_id is not None: + intermediate_executors.add(event.executor_id) + + # The default-final list ([C]) is implicitly narrowed by the intermediate designation, + # so no participant surfaces as terminal output and C surfaces as intermediate. + assert output_executors == set() + assert intermediate_executors == {"C"} + + +@pytest.mark.asyncio +async def test_sequential_get_outputs_returns_terminator_only() -> None: + """WorkflowRunResult.get_outputs() returns only the terminator's yield.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + + workflow = SequentialBuilder(participants=[a, b]).build() + result = await workflow.run("hi") + outputs = result.get_outputs() + assert len(outputs) == 1 + + +# --------------------------------------------------------------------------- +# Concurrent +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_concurrent_default_only_aggregator_is_output() -> None: + """Default Concurrent designates only the aggregator; participants are hidden.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + + workflow = ConcurrentBuilder(participants=[a, b]).build() + + output_events: list[Any] = [] + intermediate_events: list[Any] = [] + async for event in workflow.run("hello", stream=True): + if event.type == "output": + output_events.append(event) + elif event.type == "intermediate": + intermediate_events.append(event) + + # Aggregator is the only designated executor → only it emits type='output'. + assert len(output_events) == 1 + + assert not intermediate_events + + +@pytest.mark.asyncio +async def test_concurrent_output_from_designates_workflow_output_participants() -> None: + """Concurrent output_from designates participant outputs alongside the aggregator.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + + workflow = ConcurrentBuilder(participants=[a, b], output_from=[a, "B"]).build() + result = await workflow.run("hello") + outputs = result.get_outputs() + assert len(outputs) == 3 + + +@pytest.mark.asyncio +async def test_concurrent_intermediate_output_from_surface_as_intermediate() -> None: + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + + workflow = ConcurrentBuilder(participants=[a, b], intermediate_output_from=["A", b]).build() + + output_executors: set[str] = set() + intermediate_executors: set[str] = set() + async for event in workflow.run("hello", stream=True): + if event.type == "output" and event.executor_id is not None: + output_executors.add(event.executor_id) + elif event.type == "intermediate" and event.executor_id is not None: + intermediate_executors.add(event.executor_id) + + assert "aggregator" in output_executors + assert intermediate_executors == {"A", "B"} + + +# --------------------------------------------------------------------------- +# Sequential wrapped as_agent +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_sequential_default_as_agent_forwards_original_content_types() -> None: + """Default Sequential wrapped as_agent forwards original content types.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder(participants=[a, b, c]).build() + agent = workflow.as_agent("seq") + + response = await agent.run("hi") + + text_contents = [c for m in response.messages for c in m.contents if c.type == "text"] + reasoning_contents = [c for m in response.messages for c in m.contents if c.type == "text_reasoning"] + + assert any("C reply" in c.text for c in text_contents) + assert not reasoning_contents + + +@pytest.mark.asyncio +async def test_sequential_as_agent_output_from_all_text() -> None: + """output_from makes designated participant replies normal response text content.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder(participants=[a, b, c], output_from=["A", "B", "C"]).build() + agent = workflow.as_agent("seq") + + response = await agent.run("hi") + text_contents = [c for m in response.messages for c in m.contents if c.type == "text"] + text = " ".join(c.text for c in text_contents) + assert "A reply" in text + assert "B reply" in text + assert "C reply" in text + + +@pytest.mark.asyncio +async def test_sequential_as_agent_intermediate_output_from_keeps_text_content() -> None: + """intermediate_output_from keeps selected participant replies as their original content type.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder(participants=[a, b, c], intermediate_output_from=["A", "B"]).build() + agent = workflow.as_agent("seq") + + response = await agent.run("hi") + + text_contents = [c for m in response.messages for c in m.contents if c.type == "text"] + reasoning_contents = [c for m in response.messages for c in m.contents if c.type == "text_reasoning"] + assert any("C reply" in c.text for c in text_contents) + assert any("A reply" in c.text for c in text_contents) + assert any("B reply" in c.text for c in text_contents) + assert not reasoning_contents + + +# --------------------------------------------------------------------------- +# Concurrent wrapped as_agent +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_concurrent_default_as_agent_participants_keep_text_content() -> None: + """Default Concurrent wrapped as_agent keeps original participant content types.""" + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + + workflow = ConcurrentBuilder(participants=[a, b]).build() + agent = workflow.as_agent("concurrent") + + response = await agent.run("hi") + + text_contents = [c for m in response.messages for c in m.contents if c.type == "text"] + reasoning_contents = [c for m in response.messages for c in m.contents if c.type == "text_reasoning"] + + assert not any("A reply" in c.text for c in reasoning_contents) + assert not any("B reply" in c.text for c in reasoning_contents) + + # The aggregator's default-yielded AgentResponse passes through as text content. + assert text_contents, "expected at least one terminal text content from the aggregator" + + +# --------------------------------------------------------------------------- +# GroupChat +# --------------------------------------------------------------------------- + + +def _two_step_selector() -> Callable[[GroupChatState], str]: + """Selector that picks each participant once, then keeps the first to keep tests bounded.""" + counter = {"n": 0} + + def _select(state: GroupChatState) -> str: + participants = list(state.participants.keys()) + step = counter["n"] + counter["n"] = step + 1 + if step == 0: + return participants[0] + if step == 1 and len(participants) > 1: + return participants[1] + return participants[0] + + return _select + + +@pytest.mark.asyncio +async def test_group_chat_default_only_orchestrator_is_output() -> None: + """Default GroupChat designates only the orchestrator; participant replies are hidden.""" + alpha = _EchoAgent(name="alpha") + beta = _EchoAgent(name="beta") + + workflow = GroupChatBuilder( + participants=[alpha, beta], + max_rounds=2, + selection_func=_two_step_selector(), + ).build() + + output_executors: set[str] = set() + intermediate_executors: set[str] = set() + async for event in workflow.run("kickoff", stream=True): + if event.type == "output" and event.executor_id is not None: + output_executors.add(event.executor_id) + elif event.type == "intermediate" and event.executor_id is not None: + intermediate_executors.add(event.executor_id) + + assert "group_chat_orchestrator" in output_executors + assert "alpha" not in intermediate_executors + assert "beta" not in intermediate_executors + # Participants must NOT appear among designated outputs in the default contract. + assert "alpha" not in output_executors + assert "beta" not in output_executors + + +@pytest.mark.asyncio +async def test_group_chat_output_from_designates_workflow_output_participants() -> None: + """GroupChat output_from designates participants alongside the orchestrator.""" + alpha = _EchoAgent(name="alpha") + beta = _EchoAgent(name="beta") + + workflow = GroupChatBuilder( + participants=[alpha, beta], + max_rounds=2, + selection_func=_two_step_selector(), + output_from=[alpha, "beta"], + ).build() + + output_executors: set[str] = set() + async for event in workflow.run("kickoff", stream=True): + if event.type == "output" and event.executor_id is not None: + output_executors.add(event.executor_id) + + assert {"group_chat_orchestrator", "alpha", "beta"}.issubset(output_executors) + + +@pytest.mark.asyncio +async def test_group_chat_intermediate_output_from_surface_as_intermediate() -> None: + alpha = _EchoAgent(name="alpha") + beta = _EchoAgent(name="beta") + + workflow = GroupChatBuilder( + participants=[alpha, beta], + max_rounds=2, + selection_func=_two_step_selector(), + intermediate_output_from=["alpha", beta], + ).build() + + output_executors: set[str] = set() + intermediate_executors: set[str] = set() + async for event in workflow.run("kickoff", stream=True): + if event.type == "output" and event.executor_id is not None: + output_executors.add(event.executor_id) + elif event.type == "intermediate" and event.executor_id is not None: + intermediate_executors.add(event.executor_id) + + assert "group_chat_orchestrator" in output_executors + assert intermediate_executors == {"alpha", "beta"} + + +# --------------------------------------------------------------------------- +# Handoff +# --------------------------------------------------------------------------- + + +def test_handoff_builder_designates_every_participant_as_output() -> None: + """Handoff has no intermediate channel — every participant's reply is a primary + output. The builder must designate all participants in the workflow's + output designation so each per-agent yield surfaces as type='output'. + + Structural assertion (vs end-to-end) because Handoff agents require a full + chat-client/middleware stack that we don't want to reproduce in this contract test. + """ + from agent_framework import Agent + from agent_framework._clients import BaseChatClient + from agent_framework._middleware import ChatMiddlewareLayer + from agent_framework._tools import FunctionInvocationLayer + + class _StubClient(FunctionInvocationLayer[Any], ChatMiddlewareLayer[Any], BaseChatClient[Any]): + def __init__(self) -> None: + ChatMiddlewareLayer.__init__(self) + FunctionInvocationLayer.__init__(self) + BaseChatClient.__init__(self) + + def _inner_get_response(self, **kwargs: Any) -> Any: # pragma: no cover - never called + raise NotImplementedError + + alpha = Agent( + name="alpha", + id="alpha", + client=_StubClient(), + require_per_service_call_history_persistence=True, + ) + beta = Agent( + name="beta", + id="beta", + client=_StubClient(), + require_per_service_call_history_persistence=True, + ) + + workflow = HandoffBuilder(participants=[alpha, beta]).with_start_agent(alpha).build() + + designated = {ex.id for ex in workflow.get_output_executors()} + assert "alpha" in designated, f"alpha must be designated; got {designated}" + assert "beta" in designated, f"beta must be designated; got {designated}" + + +def test_handoff_builder_output_from_can_select_workflow_output_participants() -> None: + from agent_framework import Agent + from agent_framework._clients import BaseChatClient + from agent_framework._middleware import ChatMiddlewareLayer + from agent_framework._tools import FunctionInvocationLayer + + class _StubClient(FunctionInvocationLayer[Any], ChatMiddlewareLayer[Any], BaseChatClient[Any]): + def __init__(self) -> None: + ChatMiddlewareLayer.__init__(self) + FunctionInvocationLayer.__init__(self) + BaseChatClient.__init__(self) + + def _inner_get_response(self, **kwargs: Any) -> Any: # pragma: no cover - never called + raise NotImplementedError + + alpha = Agent( + name="alpha", + id="alpha", + client=_StubClient(), + require_per_service_call_history_persistence=True, + ) + beta = Agent( + name="beta", + id="beta", + client=_StubClient(), + require_per_service_call_history_persistence=True, + ) + + workflow = HandoffBuilder(participants=[alpha, beta], output_from=["alpha"]).with_start_agent(alpha).build() + + assert {ex.id for ex in workflow.get_output_executors()} == {"alpha"} + + +def test_handoff_builder_intermediate_output_from_demotes_from_default_output() -> None: + """Regression: `intermediate_output_from` alone must not collide with the default output list. + + Handoff defaults workflow output to every participant. Before the fix, supplying + `intermediate_output_from=["alpha"]` without restating `output_from` triggered + "Participants cannot be both output and intermediate designated: ['alpha']" because + alpha was simultaneously in the default output list and the explicit intermediate list. + The contract documented at `_handoff.py:619-622` promises `intermediate_output_from` is + usable on its own. + """ + from agent_framework import Agent + from agent_framework._clients import BaseChatClient + from agent_framework._middleware import ChatMiddlewareLayer + from agent_framework._tools import FunctionInvocationLayer + + class _StubClient(FunctionInvocationLayer[Any], ChatMiddlewareLayer[Any], BaseChatClient[Any]): + def __init__(self) -> None: + ChatMiddlewareLayer.__init__(self) + FunctionInvocationLayer.__init__(self) + BaseChatClient.__init__(self) + + def _inner_get_response(self, **kwargs: Any) -> Any: # pragma: no cover - never called + raise NotImplementedError + + alpha = Agent(name="alpha", id="alpha", client=_StubClient(), require_per_service_call_history_persistence=True) + beta = Agent(name="beta", id="beta", client=_StubClient(), require_per_service_call_history_persistence=True) + + workflow = ( + HandoffBuilder(participants=[alpha, beta], intermediate_output_from=["alpha"]).with_start_agent(alpha).build() + ) + + # alpha is implicitly removed from the default-final set; beta remains final. + assert {ex.id for ex in workflow.get_output_executors()} == {"beta"} + assert {ex.id for ex in workflow.get_intermediate_executors()} == {"alpha"} + + +# --------------------------------------------------------------------------- +# Magentic +# --------------------------------------------------------------------------- + + +class _StubMagenticManager(MagenticManagerBase): + """Deterministic manager that finishes after one round with a fixed final answer.""" + + FINAL_ANSWER: ClassVar[str] = "MAGENTIC_FINAL" + + def __init__(self) -> None: + super().__init__(max_stall_count=3) + self.name = "magentic_manager" + self.next_speaker_name = "alpha" + + async def plan(self, magentic_context: MagenticContext) -> Message: + return Message("assistant", ["Plan: do the thing."], author_name=self.name) + + async def replan(self, magentic_context: MagenticContext) -> Message: + return Message("assistant", ["Replan."], author_name=self.name) + + async def create_progress_ledger(self, magentic_context: MagenticContext) -> MagenticProgressLedger: + is_satisfied = len(magentic_context.chat_history) > 1 + return MagenticProgressLedger( + is_request_satisfied=MagenticProgressLedgerItem(reason="t", answer=is_satisfied), + is_in_loop=MagenticProgressLedgerItem(reason="t", answer=False), + is_progress_being_made=MagenticProgressLedgerItem(reason="t", answer=True), + next_speaker=MagenticProgressLedgerItem(reason="t", answer=self.next_speaker_name), + instruction_or_question=MagenticProgressLedgerItem(reason="t", answer="Go."), + ) + + async def prepare_final_answer(self, magentic_context: MagenticContext) -> Message: + return Message("assistant", [self.FINAL_ANSWER], author_name=self.name) + + +def test_magentic_builder_default_only_manager_designated() -> None: + """Default Magentic: only the orchestrator (manager) is designated for terminal output; + participant replies surface as type='intermediate'. + + Structural assertion on the workflow's output designation because exercising a Magentic + plan/replan loop end-to-end is heavy and orthogonal to this contract. + """ + manager = _StubMagenticManager() + alpha = _EchoAgent(name="alpha") + + workflow = MagenticBuilder(participants=[alpha], manager=manager).build() + + designated = {ex.id for ex in workflow.get_output_executors()} + assert "magentic_orchestrator" in designated, f"manager must be designated; got {designated}" + assert "alpha" not in designated, f"participant must not be designated by default; got {designated}" + + +def test_magentic_builder_output_from_designates_workflow_output_participants() -> None: + """Magentic output_from designates workers alongside the orchestrator.""" + manager = _StubMagenticManager() + alpha = _EchoAgent(name="alpha") + + workflow = MagenticBuilder(participants=[alpha], manager=manager, output_from=["alpha"]).build() + + designated = {ex.id for ex in workflow.get_output_executors()} + assert {"magentic_orchestrator", "alpha"}.issubset(designated) + + +def test_magentic_builder_intermediate_output_from_designates_intermediate_workers() -> None: + manager = _StubMagenticManager() + alpha = _EchoAgent(name="alpha") + + workflow = MagenticBuilder(participants=[alpha], manager=manager, intermediate_output_from=[alpha]).build() + + assert {ex.id for ex in workflow.get_output_executors()} == {"magentic_orchestrator"} + assert {ex.id for ex in workflow.get_intermediate_executors()} == {"alpha"} + + +def test_sequential_output_from_all_selects_all_participants() -> None: + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder(participants=[a, b, c], output_from="all").build() + + assert {ex.id for ex in workflow.get_output_executors()} == {"A", "B", "C"} + + +def test_sequential_intermediate_output_from_all_other_selects_non_outputs() -> None: + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + c = _EchoAgent(name="C") + + workflow = SequentialBuilder( + participants=[a, b, c], output_from=["C"], intermediate_output_from="all_other" + ).build() + + assert {ex.id for ex in workflow.get_output_executors()} == {"C"} + assert {ex.id for ex in workflow.get_intermediate_executors()} == {"A", "B"} + + +def test_sequential_all_other_with_omitted_output_from_selects_all_intermediate() -> None: + a = _EchoAgent(name="A") + b = _EchoAgent(name="B") + + workflow = SequentialBuilder(participants=[a, b], intermediate_output_from="all_other").build() + + assert {ex.id for ex in workflow.get_output_executors()} == set() + assert {ex.id for ex in workflow.get_intermediate_executors()} == {"A", "B"} + + +# --------------------------------------------------------------------------- +# Participant designation validation +# --------------------------------------------------------------------------- + + +def _build_sequential_with_designation(**kwargs: Any) -> None: + SequentialBuilder(participants=[_EchoAgent(name="alpha"), _EchoAgent(name="beta")], **kwargs).build() + + +def _build_concurrent_with_designation(**kwargs: Any) -> None: + ConcurrentBuilder(participants=[_EchoAgent(name="alpha"), _EchoAgent(name="beta")], **kwargs).build() + + +def _build_group_chat_with_designation(**kwargs: Any) -> None: + GroupChatBuilder( + participants=[_EchoAgent(name="alpha"), _EchoAgent(name="beta")], + max_rounds=1, + selection_func=_two_step_selector(), + **kwargs, + ).build() + + +def _build_magentic_with_designation(**kwargs: Any) -> None: + MagenticBuilder(participants=[_EchoAgent(name="alpha")], manager=_StubMagenticManager(), **kwargs).build() + + +def _build_handoff_with_designation(**kwargs: Any) -> None: + from agent_framework import Agent + from agent_framework._clients import BaseChatClient + from agent_framework._middleware import ChatMiddlewareLayer + from agent_framework._tools import FunctionInvocationLayer + + class _StubClient(FunctionInvocationLayer[Any], ChatMiddlewareLayer[Any], BaseChatClient[Any]): + def __init__(self) -> None: + ChatMiddlewareLayer.__init__(self) + FunctionInvocationLayer.__init__(self) + BaseChatClient.__init__(self) + + def _inner_get_response(self, **kwargs: Any) -> Any: # pragma: no cover - never called + raise NotImplementedError + + alpha = Agent( + name="alpha", + id="alpha", + client=_StubClient(), + require_per_service_call_history_persistence=True, + ) + beta = Agent( + name="beta", + id="beta", + client=_StubClient(), + require_per_service_call_history_persistence=True, + ) + HandoffBuilder(participants=[alpha, beta], **kwargs).with_start_agent(alpha).build() + + +@pytest.mark.parametrize( + "build", + [ + _build_sequential_with_designation, + _build_concurrent_with_designation, + _build_group_chat_with_designation, + _build_magentic_with_designation, + _build_handoff_with_designation, + ], +) +@pytest.mark.parametrize( + ("kwargs", "match"), + [ + ({"output_from": [], "intermediate_output_from": []}, "cannot both be empty"), + ({"output_from": ["alpha", "alpha"]}, "Duplicate output participant"), + ({"output_from": ["alpha"], "intermediate_output_from": ["alpha"]}, "cannot be both output"), + ({"output_from": ["missing"]}, "Unknown output participant"), + ({"output_from": "all_other"}, "output_from='all_other'"), + ], +) +def test_participant_output_config_validation(build: Callable[..., None], kwargs: dict[str, Any], match: str) -> None: + with pytest.raises(ValueError, match=match): + build(**kwargs) + + +@pytest.mark.parametrize( + "build", + [ + _build_sequential_with_designation, + _build_concurrent_with_designation, + _build_group_chat_with_designation, + _build_magentic_with_designation, + _build_handoff_with_designation, + ], +) +def test_participant_output_config_rejects_final_output_from_parameter(build: Callable[..., None]) -> None: + with pytest.raises(TypeError, match="final_output_from"): + build(final_output_from=["beta"]) diff --git a/python/samples/03-workflows/README.md b/python/samples/03-workflows/README.md index d4e6fc1fb1..c79203d742 100644 --- a/python/samples/03-workflows/README.md +++ b/python/samples/03-workflows/README.md @@ -89,6 +89,7 @@ Write workflows as plain Python async functions — no graph concepts, no execut | Multi-Selection Edge Group | [control-flow/multi_selection_edge_group.py](./control-flow/multi_selection_edge_group.py) | Select one or many targets dynamically (subset fan-out) | | Simple Loop | [control-flow/simple_loop.py](./control-flow/simple_loop.py) | Feedback loop where an agent judges ABOVE/BELOW/MATCHED | | Workflow Cancellation | [control-flow/workflow_cancellation.py](./control-flow/workflow_cancellation.py) | Cancel a running workflow using asyncio tasks | +| Workflow and Intermediate Outputs | [control-flow/intermediate_vs_terminal_outputs.py](./control-flow/intermediate_vs_terminal_outputs.py) | Select Workflow Output and Intermediate Output executors; hide unselected yields; map Intermediate Output events to `text_reasoning` content via `as_agent` | ### human-in-the-loop @@ -118,6 +119,43 @@ For additional observability samples in Agent Framework, see the [observability Orchestration-focused samples (Sequential, Concurrent, Handoff, GroupChat, Magentic), including builder-based `workflow.as_agent(...)` variants, are documented in the [orchestrations](./orchestrations/README.md) directory. +### output selection + +Workflow Output selection controls which `ctx.yield_output(...)` calls are visible to callers as `type='output'` +events and through `WorkflowRunResult.get_outputs()`. The core rule is that `output_from` is an allow-list for +Workflow Output, not a routing rule for every other executor output. Unselected executor payloads are hidden unless +`intermediate_output_from` explicitly selects them as Intermediate Output. + +Use `output_from` and `intermediate_output_from` as the canonical API: + +| Selection | Workflow Output | Intermediate Output | Hidden payloads | +| --- | --- | --- | --- | +| Omit both selections | Every executor `yield_output`; emits a deprecation warning | None | None | +| `output_from="all"` | Every executor `yield_output`; no warning | None | None | +| `output_from=[answerer]` | Only `answerer` | None | All other executor payloads | +| `output_from=[answerer], intermediate_output_from="all_other"` | Only `answerer` | Every output-capable executor not selected by `output_from` | None | +| `intermediate_output_from="all_other"` | None | Every output-capable executor | None | +| `output_from=[], intermediate_output_from="all_other"` | None | Every output-capable executor | None | +| `output_from=[answerer], intermediate_output_from=[planner, researcher]` | Only `answerer` | `planner` and `researcher` | Any other executor payloads | + +Invalid selections fail at construction or build time: + +| Invalid selection | Why it fails | +| --- | --- | +| `output_from="all_other"` | `"all_other"` is only valid for `intermediate_output_from` | +| `intermediate_output_from="all"` | `"all"` is only valid for `output_from` | +| The same executor in both selections | One payload cannot be both Workflow Output and Intermediate Output | +| Duplicate executor selections | Duplicates are treated as configuration errors | +| Unknown executor selections | Typos and missing participants are rejected | +| `output_from=[], intermediate_output_from=[]` | Both explicit selections are empty | + +Compatibility aliases such as `output_executors` emit deprecation warnings where supported. New samples and +applications should use `output_from` and `intermediate_output_from`. + +When a workflow is wrapped with `workflow.as_agent()`, Workflow Output becomes normal agent text content. Intermediate +Output becomes `text_reasoning` content, so `AgentResponse.text` remains focused on the caller-facing answer while +callers can still inspect progress or supporting work from the response messages. + ### parallelism | Sample | File | Concepts | @@ -174,7 +212,7 @@ Sequential orchestration uses a few small adapter nodes for plumbing: - "input-conversation" normalizes input to `list[Message]` - "to-conversation:" converts agent responses into the shared conversation -- "complete" publishes the final output event (type='output') +- "complete" publishes the Workflow Output event (`type='output'`) These may appear in event streams (executor_invoked/executor_completed). They're analogous to concurrent’s dispatcher and aggregator and can be ignored if you only care about agent activity. diff --git a/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py b/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py index b503f7574f..b156858484 100644 --- a/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/group_chat_workflow_as_agent.py @@ -54,11 +54,11 @@ async def main() -> None: credential=AzureCliCredential(), ) - # intermediate_outputs=True: Enable intermediate outputs to observe the conversation as it unfolds - # (Intermediate outputs will be emitted as WorkflowOutputEvent events) + # Mark participant responses as intermediate so workflow.as_agent() maps + # them to text_reasoning content while the final answer remains normal text. workflow = GroupChatBuilder( participants=[researcher, writer], - intermediate_outputs=True, + intermediate_output_from=[researcher, writer], orchestrator_agent=Agent( client=_orch_client, name="Orchestrator", diff --git a/python/samples/03-workflows/agents/magentic_workflow_as_agent.py b/python/samples/03-workflows/agents/magentic_workflow_as_agent.py index 6cc91a9dcd..488cd91f20 100644 --- a/python/samples/03-workflows/agents/magentic_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/magentic_workflow_as_agent.py @@ -72,11 +72,11 @@ async def main() -> None: print("\nBuilding Magentic Workflow...") - # intermediate_outputs=True: Enable intermediate outputs to observe the conversation as it unfolds - # (Intermediate outputs will be emitted as WorkflowOutputEvent events) + # Mark participant responses as intermediate so workflow.as_agent() maps + # them to text_reasoning content while the final answer remains normal text. workflow = MagenticBuilder( participants=[researcher_agent, coder_agent], - intermediate_outputs=True, + intermediate_output_from=[researcher_agent, coder_agent], manager_agent=manager_agent, max_round_count=10, max_stall_count=3, diff --git a/python/samples/03-workflows/agents/sequential_workflow_as_agent.py b/python/samples/03-workflows/agents/sequential_workflow_as_agent.py index 120bd448aa..69dfecd410 100644 --- a/python/samples/03-workflows/agents/sequential_workflow_as_agent.py +++ b/python/samples/03-workflows/agents/sequential_workflow_as_agent.py @@ -77,9 +77,9 @@ async def main() -> None: Note: `workflow.as_agent()` returns ONLY the final agent's response (the "answer") — the prior agents' work - is not included in the response. To observe intermediate agents while running as an agent, build with - `SequentialBuilder(participants=[...], intermediate_outputs=True)`; the intermediate replies are then - surfaced as `data` events and merged into the AgentResponse. + is not included in the response. To preserve earlier participant replies while running as an agent, build with + `SequentialBuilder(participants=[...], intermediate_output_from=[writer])`; intermediate workflow events become + `text_reasoning` content on the AgentResponse, while `.text` remains terminal-output only. """ diff --git a/python/samples/03-workflows/control-flow/intermediate_vs_terminal_outputs.py b/python/samples/03-workflows/control-flow/intermediate_vs_terminal_outputs.py new file mode 100644 index 0000000000..520c17a07b --- /dev/null +++ b/python/samples/03-workflows/control-flow/intermediate_vs_terminal_outputs.py @@ -0,0 +1,156 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +from agent_framework import ( + Message, + WorkflowBuilder, + WorkflowContext, + WorkflowExecutor, + executor, +) +from typing_extensions import Never + +""" +Sample: Workflow Output vs Intermediate Output labeling + +What this sample shows +- How ``WorkflowBuilder(output_from=[...])`` designates which executors emit + Workflow Output. +- How ``WorkflowBuilder(intermediate_output_from=[...])`` designates which executor + yields surface as Intermediate Output (``type='intermediate'`` events). +- How unlisted executor yields are hidden from caller-facing output/intermediate + events in explicit designation mode. +- How the same workflow wrapped via ``workflow.as_agent()`` translates intermediate + events to ``text_reasoning`` content so existing ``.text`` accessors keep + returning Workflow Output only. +- How a sub-workflow embedded via ``WorkflowExecutor`` bubbles its intermediate + emissions up through the parent's event stream, attributed to the + ``WorkflowExecutor`` id rather than the child's internal executor ids. + +The output selection contract: +- Compatibility mode: when neither ``output_from`` nor ``intermediate_output_from`` + is provided, every ``yield_output`` produces Workflow Output and a deprecation + warning points to explicit selection. +- Explicit selection mode: provide either ``output_from`` or + ``intermediate_output_from``. Executors selected by ``output_from`` emit Workflow Output + (``type='output'`` events); executors selected by ``intermediate_output_from`` emit + Intermediate Output (``type='intermediate'`` events); unselected executor yields are + hidden from the stream and ``WorkflowRunResult`` output accessors. +- Validation: explicit selections must not both be empty; duplicate executor entries, + overlap between Workflow Output and Intermediate Output, unknown executors, invalid + literals, and selected executors without workflow output types are rejected. + +Prerequisites +- No external services required. +""" + + +@executor(id="planner") +async def planner(messages: list[Message], ctx: WorkflowContext[list[Message], str]) -> None: + """Intermediate step: emits a visible progress note, then forwards.""" + prompt = messages[0].text if messages else "" + await ctx.yield_output(f"plan: starting work on '{prompt}'") + await ctx.send_message(messages) + + +@executor(id="researcher") +async def researcher(messages: list[Message], ctx: WorkflowContext[list[Message], str]) -> None: + """Intermediate step: emits visible progress, then forwards.""" + prompt = messages[0].text if messages else "" + await ctx.yield_output(f"research: gathering data for '{prompt}'") + await ctx.send_message(messages) + + +@executor(id="answerer") +async def answerer(messages: list[Message], ctx: WorkflowContext[Never, str]) -> None: + """Designated Workflow Output: emits the workflow's answer.""" + prompt = messages[0].text if messages else "" + await ctx.yield_output(f"final answer to '{prompt}': 42") + + +async def main() -> None: + # Build with explicit Workflow Output and Intermediate Output selections. + # `answerer` produces type='output' events; planner and researcher produce + # visible type='intermediate' events. + workflow = ( + WorkflowBuilder( + start_executor=planner, + output_from=[answerer], + intermediate_output_from=[planner, researcher], + ) + .add_edge(planner, researcher) + .add_edge(researcher, answerer) + .build() + ) + + initial = [Message(role="user", contents=["life, the universe, and everything"])] + + print("=== Streaming events (workflow.run(stream=True)) ===") + async for event in workflow.run(initial, stream=True): + if event.type == "intermediate": + print(f" [intermediate] {event.executor_id}: {event.data}") + elif event.type == "output": + print(f" [output] {event.executor_id}: {event.data}") + + # WorkflowRunResult.get_outputs() filters to type='output' events, so it + # only returns the selected Workflow Output yield. + print("\n=== Non-streaming run().get_outputs() ===") + result = await workflow.run(initial) + print(f" outputs: {result.get_outputs()}") + + # When the same workflow is wrapped via as_agent(), intermediate events + # surface as ``text_reasoning`` content; Workflow Output surfaces as + # ``text`` content. Existing callers reading ``response.text`` get only + # the selected Workflow Output because ``.text`` filters to text content. + print("\n=== workflow.as_agent() -- intermediate -> text_reasoning content ===") + agent = workflow.as_agent("planner-agent") + response = await agent.run("life, the universe, and everything") + print(f" response.text (Workflow Output only): {response.text!r}") + reasoning = " | ".join(c.text for m in response.messages for c in m.contents if c.type == "text_reasoning") + print(f" reasoning content (intermediates): {reasoning!r}") + + # Embed the same workflow as a node inside a larger workflow via WorkflowExecutor. + # Child intermediate emissions are forwarded to the parent's event stream with the + # WorkflowExecutor's id as the source, so outer callers don't have to know the + # child's internal executor layout. The 'intermediate' label is preserved across + # the boundary regardless of how the parent designates the WorkflowExecutor. + print("\n=== Embedding as a sub-workflow -- intermediates bubble up ===") + sub = WorkflowExecutor(workflow, id="sub") + + @executor(id="parent_sink") + async def parent_sink(message: str, ctx: WorkflowContext[Never, str]) -> None: + await ctx.yield_output(message) + + parent_workflow = WorkflowBuilder(start_executor=sub, output_from=[parent_sink]).add_edge(sub, parent_sink).build() + + async for event in parent_workflow.run(initial, stream=True): + if event.type == "intermediate": + print(f" [intermediate] {event.executor_id}: {event.data}") + elif event.type == "output": + print(f" [output] {event.executor_id}: {event.data}") + + """ + Sample output: + + === Streaming events (workflow.run(stream=True)) === + [intermediate] planner: plan: starting work on 'life, the universe, and everything' + [intermediate] researcher: research: gathering data for 'life, the universe, and everything' + [output] answerer: final answer to 'life, the universe, and everything': 42 + + === Non-streaming run().get_outputs() === + outputs: ["final answer to 'life, the universe, and everything': 42"] + + === workflow.as_agent() -- intermediate -> text_reasoning content === + response.text (Workflow Output only): "final answer to 'life, the universe, and everything': 42" + reasoning content (intermediates): "plan: starting work on ... | research: gathering data for ..." + + === Embedding as a sub-workflow -- intermediates bubble up === + [intermediate] sub: plan: starting work on 'life, the universe, and everything' + [intermediate] sub: research: gathering data for 'life, the universe, and everything' + [output] parent_sink: final answer to 'life, the universe, and everything': 42 + """ + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py b/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py index adb1fff4d5..6889914dfd 100644 --- a/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py +++ b/python/samples/03-workflows/human-in-the-loop/agents_with_approval_requests.py @@ -248,7 +248,7 @@ async def main() -> None: # Build the workflow workflow = ( - WorkflowBuilder(start_executor=email_processor, output_executors=[conclude_workflow]) + WorkflowBuilder(start_executor=email_processor, output_from=[conclude_workflow]) .add_edge(email_processor, email_writer_agent) .add_edge(email_writer_agent, conclude_workflow) .build() diff --git a/python/samples/03-workflows/orchestrations/README.md b/python/samples/03-workflows/orchestrations/README.md index 0c4406c247..c0cd65b650 100644 --- a/python/samples/03-workflows/orchestrations/README.md +++ b/python/samples/03-workflows/orchestrations/README.md @@ -81,6 +81,41 @@ from agent_framework.orchestrations import ( ## Tips +**Participant output selection**: Orchestration builders use participant-oriented names for Workflow Output selection. +Use `output_from=[...]` when participant responses should be Workflow Output (`type='output'` events), and +`intermediate_output_from=[...]` when participant responses should be Intermediate Output (`type='intermediate'` +events). `output_from` is an allow-list for Workflow Output, not a routing rule for every other participant output. +Unselected participant responses are hidden unless `intermediate_output_from` selects them. + +| Selection | Workflow Output | Intermediate Output | Hidden payloads | +| --- | --- | --- | --- | +| Omit both selections | Builder default Workflow Output contract | None | Builder-specific non-output participant payloads | +| `output_from="all"` | Every output-capable participant | None | None | +| `output_from=[writer]` | Only `writer` | None | All other participant payloads | +| `output_from=[writer], intermediate_output_from="all_other"` | Only `writer` | Every output-capable participant not selected by `output_from` | None | +| `intermediate_output_from="all_other"` | None, except builder-internal default output executors where applicable | Every output-capable participant | Builder-internal plumbing payloads | +| `output_from=[], intermediate_output_from="all_other"` | None, except builder-internal default output executors where applicable | Every output-capable participant | Builder-internal plumbing payloads | +| `output_from=[writer], intermediate_output_from=[researcher, reviewer]` | Only `writer` | `researcher` and `reviewer` | Any other participant payloads | + +Invalid selections fail at construction or build time: + +| Invalid selection | Why it fails | +| --- | --- | +| `output_from="all_other"` | `"all_other"` is only valid for `intermediate_output_from` | +| `intermediate_output_from="all"` | `"all"` is only valid for `output_from` | +| The same participant in both selections | One payload cannot be both Workflow Output and Intermediate Output | +| Duplicate participant selections | Duplicates are treated as configuration errors | +| Unknown participant selections | Typos and missing participants are rejected | +| `output_from=[], intermediate_output_from=[]` | Both explicit selections are empty | + +By default, Sequential keeps the last participant as Workflow Output. Concurrent, GroupChat, and Magentic keep their +synthetic aggregator/orchestrator/manager executors as Workflow Output, while participant responses stay hidden unless +selected. Handoff keeps participants as Workflow Output by default. + +When an orchestration workflow is exposed via `workflow.as_agent()`, Workflow Output becomes normal text content in +the `AgentResponse`; Intermediate Output becomes `text_reasoning` content. This preserves `.text` while making +selected progress available for callers that inspect message contents. + **Magentic checkpointing tip**: Treat `MagenticBuilder.participants` keys as stable identifiers. When resuming from a checkpoint, the rebuilt workflow must reuse the same participant names; otherwise the checkpoint cannot be applied and the run will fail fast. **Handoff workflow tip**: Handoff workflows maintain the full conversation history including any `Message.additional_properties` emitted by your agents. This ensures routing metadata remains intact across all agent transitions. For specialist-to-specialist handoffs, use `.add_handoff(source, targets)` to configure which agents can route to which others with a fluent, type-safe API. @@ -90,7 +125,7 @@ from agent_framework.orchestrations import ( **Sequential orchestration note**: Sequential orchestration uses a few small adapter nodes for plumbing: - `input-conversation` normalizes input to `list[Message]` - `to-conversation:` converts agent responses into the shared conversation -- `complete` publishes the final output event (type='output') +- `complete` publishes the Workflow Output event (`type='output'`) These may appear in event streams (executor_invoked/executor_completed). They're analogous to concurrent's dispatcher and aggregator and can be ignored if you only care about agent activity. diff --git a/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py b/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py index dea82a4352..079f66ba5c 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py +++ b/python/samples/03-workflows/orchestrations/group_chat_agent_manager.py @@ -78,13 +78,14 @@ async def main() -> None: # Build the group chat workflow # termination_condition: stop after 4 assistant messages # (The agent orchestrator will intelligently decide when to end before this limit but just in case) - # intermediate_outputs=True: Enable intermediate outputs to observe the conversation as it unfolds - # (Intermediate outputs will be emitted as WorkflowOutputEvent events) + # Mark participant responses as intermediate so the stream shows the + # conversation as it unfolds while the orchestrator's transcript remains the + # terminal workflow output. workflow = ( GroupChatBuilder( participants=[researcher, writer], termination_condition=lambda messages: sum(1 for msg in messages if msg.role == "assistant") >= 4, - intermediate_outputs=True, + intermediate_output_from=[researcher, writer], orchestrator_agent=orchestrator_agent, ) # Set a hard termination condition: stop after 4 assistant messages @@ -102,7 +103,7 @@ async def main() -> None: # Keep track of the last response to format output nicely in streaming mode last_response_id: str | None = None async for event in workflow.run(task, stream=True): - if event.type == "output": + if event.type in ("intermediate", "output"): data = event.data if isinstance(data, AgentResponseUpdate): rid = data.response_id diff --git a/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py b/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py index 867bbd7bc3..259bd7bd5a 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py +++ b/python/samples/03-workflows/orchestrations/group_chat_philosophical_debate.py @@ -219,13 +219,16 @@ Share your perspective authentically. Feel free to: ) # termination_condition: stop after 10 assistant messages - # intermediate_outputs=True: Enable intermediate outputs to observe the conversation as it unfolds - # (Intermediate outputs will be emitted as WorkflowOutputEvent events) + # Mark participant responses as intermediate so the stream shows the + # conversation as it unfolds while the orchestrator's transcript remains the + # terminal workflow output. workflow = ( GroupChatBuilder( participants=[farmer, developer, teacher, activist, spiritual_leader, artist, immigrant, doctor], termination_condition=lambda messages: sum(1 for msg in messages if msg.role == "assistant") >= 10, - intermediate_outputs=True, + intermediate_output_from=[ + "all", + ], orchestrator_agent=moderator, ) .with_termination_condition(lambda messages: sum(1 for msg in messages if msg.role == "assistant") >= 10) @@ -254,7 +257,7 @@ Share your perspective authentically. Feel free to: # Keep track of the last response to format output nicely in streaming mode last_response_id: str | None = None async for event in workflow.run(f"Please begin the discussion on: {topic}", stream=True): - if event.type == "output": + if event.type in ("intermediate", "output"): data = event.data if isinstance(data, AgentResponseUpdate): rid = data.response_id diff --git a/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py b/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py index 2fceaa98d0..fb20dcca59 100644 --- a/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py +++ b/python/samples/03-workflows/orchestrations/group_chat_simple_selector.py @@ -96,13 +96,14 @@ async def main() -> None: # This will end the conversation after the expert has spoken 2 times (one iteration loop) # Note: it's possible that the expert gets it right the first time and the other participants # have nothing to add, but for demo purposes we want to see at least one full round of interaction. - # intermediate_outputs=True: Enable intermediate outputs to observe the conversation as it unfolds - # (Intermediate outputs will be emitted as WorkflowOutputEvent events) + # Mark participant responses as intermediate so the stream shows the + # conversation as it unfolds while the orchestrator's transcript remains the + # terminal workflow output. workflow = ( GroupChatBuilder( participants=[expert, verifier, clarifier, skeptic], termination_condition=lambda conversation: len(conversation) >= 6, - intermediate_outputs=True, + intermediate_output_from=[expert, verifier, clarifier, skeptic], selection_func=round_robin_selector, ) # Set a hard termination condition: stop after 6 messages (user task + one full rounds + 1) @@ -123,7 +124,7 @@ async def main() -> None: # Keep track of the last response to format output nicely in streaming mode last_response_id: str | None = None async for event in workflow.run(task, stream=True): - if event.type == "output": + if event.type in ("intermediate", "output"): data = event.data if isinstance(data, AgentResponseUpdate): rid = data.response_id diff --git a/python/samples/03-workflows/orchestrations/magentic.py b/python/samples/03-workflows/orchestrations/magentic.py index f7a472049c..f750626955 100644 --- a/python/samples/03-workflows/orchestrations/magentic.py +++ b/python/samples/03-workflows/orchestrations/magentic.py @@ -88,11 +88,12 @@ async def main() -> None: print("\nBuilding Magentic Workflow...") - # intermediate_outputs=True: Enable intermediate outputs to observe the conversation as it unfolds - # (Intermediate outputs will be emitted as WorkflowOutputEvent events) + # Mark participant responses as intermediate so the stream shows the + # conversation as it unfolds while the manager's final answer remains the + # terminal workflow output. workflow = MagenticBuilder( participants=[researcher_agent, coder_agent], - intermediate_outputs=True, + intermediate_output_from=[researcher_agent, coder_agent], manager_agent=manager_agent, max_round_count=10, max_stall_count=3, @@ -115,7 +116,7 @@ async def main() -> None: last_response_id: str | None = None output_event: WorkflowEvent | None = None async for event in workflow.run(task, stream=True): - if event.type == "output" and isinstance(event.data, AgentResponseUpdate): + if event.type in ("intermediate", "output") and isinstance(event.data, AgentResponseUpdate): response_id = event.data.response_id if response_id != last_response_id: if last_response_id is not None: diff --git a/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py b/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py index e44e2a44ca..e6e254ed0d 100644 --- a/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py +++ b/python/samples/03-workflows/orchestrations/magentic_human_plan_review.py @@ -55,7 +55,7 @@ async def process_event_stream(stream: AsyncIterable[WorkflowEvent]) -> dict[str if event.type == "request_info" and event.request_type is MagenticPlanReviewRequest: requests[event.request_id] = cast(MagenticPlanReviewRequest, event.data) - if event.type == "output": + if event.type in ("intermediate", "output"): data = event.data if isinstance(data, AgentResponseUpdate): rid = data.response_id @@ -129,13 +129,14 @@ async def main() -> None: print("\nBuilding Magentic Workflow with Human Plan Review...") - # enable_plan_review=True: Request human input for plan review - # intermediate_outputs=True: Enable intermediate outputs to observe the conversation as it unfolds - # (Intermediate outputs will be emitted as WorkflowOutputEvent events) + # enable_plan_review=True: Request human input for plan review. + # Mark participant responses as intermediate so the stream shows the + # conversation as it unfolds while the manager's final answer remains the + # terminal workflow output. workflow = MagenticBuilder( participants=[researcher_agent, analyst_agent], enable_plan_review=True, - intermediate_outputs=True, + intermediate_output_from=[researcher_agent, analyst_agent], manager_agent=manager_agent, max_round_count=10, max_stall_count=1, diff --git a/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py b/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py index f4723a205d..d28fc49935 100644 --- a/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py +++ b/python/samples/03-workflows/orchestrations/sequential_chain_only_agent_responses.py @@ -66,13 +66,13 @@ async def main() -> None: workflow = SequentialBuilder( participants=[writer, translator, reviewer], chain_only_agent_responses=True, - intermediate_outputs=True, + intermediate_output_from=[writer, translator], ).build() # 3) Run and collect outputs last_agent: str | None = None async for event in workflow.run("Write a tagline for a budget-friendly eBike.", stream=True): - if event.type == "output" and isinstance(event.data, AgentResponseUpdate): + if event.type in ("intermediate", "output") and isinstance(event.data, AgentResponseUpdate): if event.data.author_name != last_agent: last_agent = event.data.author_name print() diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/05_workflows/main.py b/python/samples/04-hosting/foundry-hosted-agents/responses/05_workflows/main.py index d70edbc7bf..5a2f2d6526 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/05_workflows/main.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/05_workflows/main.py @@ -52,9 +52,9 @@ def main(): workflow_agent = ( WorkflowBuilder( start_executor=writer_executor, - # Limiting the output to only the final formatted result. - # If this is not set, all intermediate results will be included in the output. - output_executors=[format_executor], + # Select only the formatted result as Workflow Output. + # Unselected executor payloads are hidden unless selected as Intermediate Output. + output_from=[format_executor], ) .add_edge(writer_executor, legal_executor) .add_edge(legal_executor, format_executor) diff --git a/python/samples/README.md b/python/samples/README.md index 0ff8563933..6017d578f6 100644 --- a/python/samples/README.md +++ b/python/samples/README.md @@ -8,7 +8,7 @@ This directory contains samples demonstrating the capabilities of Microsoft Agen |--------|-------------| | [`01-get-started/`](./01-get-started/) | Progressive tutorial: hello agent → hosting | | [`02-agents/`](./02-agents/) | Deep-dive by concept: tools, middleware, providers, orchestrations | -| [`03-workflows/`](./03-workflows/) | Workflow patterns: sequential, concurrent, state, declarative | +| [`03-workflows/`](./03-workflows/) | Workflow patterns: sequential, concurrent, state, declarative, explicit output designation | | [`04-hosting/`](./04-hosting/) | Deployment: Azure Functions, Durable Tasks, A2A | | [`05-end-to-end/`](./05-end-to-end/) | Full applications, evaluation, demos | diff --git a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py index 51252a1786..89613072d8 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py +++ b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py @@ -248,13 +248,13 @@ async def run_agent_framework_example(task: str) -> str: participants=[researcher, planner], orchestrator_agent=Agent(client=client), max_rounds=8, - intermediate_outputs=True, + intermediate_output_from=[researcher, planner], ).build() output_messages: list[Message] = [] last_message_id: str | None = None async for event in workflow.run(task, stream=True): - if event.type == "output": + if event.type in ("intermediate", "output"): if isinstance(event.data, AgentResponseUpdate): if event.data.message_id != last_message_id: last_message_id = event.data.message_id diff --git a/python/samples/semantic-kernel-migration/orchestrations/magentic.py b/python/samples/semantic-kernel-migration/orchestrations/magentic.py index dcf7b1af33..4ce62492e2 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/magentic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/magentic.py @@ -164,13 +164,13 @@ async def run_agent_framework_example(prompt: str) -> str | None: workflow = MagenticBuilder( participants=[researcher, coder], manager_agent=manager_agent, # type: ignore - intermediate_outputs=True, + intermediate_output_from=[researcher, coder], ).build() output_messages: list[Message] = [] last_message_id: str | None = None async for event in workflow.run(prompt, stream=True): - if event.type == "output": + if event.type in ("intermediate", "output"): if isinstance(event.data, AgentResponseUpdate): if event.data.message_id != last_message_id: last_message_id = event.data.message_id diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index f9356bbdd8..01af408097 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -17,8 +17,6 @@ from agent_framework.github import GitHubCopilotAgent from copilot.generated.session_events import PermissionRequest from copilot.session import PermissionRequestResult from pydantic import BaseModel -from typing_extensions import Never - from sample_validation.const import WORKER_COMPLETED from sample_validation.discovery import DiscoveryResult from sample_validation.models import ( @@ -29,6 +27,7 @@ from sample_validation.models import ( ValidationConfig, WorkflowCreationResult, ) +from typing_extensions import Never logger = logging.getLogger(__name__) @@ -249,7 +248,7 @@ class CollectorExecutor(Executor): batch_completion: BatchCompletion, ctx: WorkflowContext[Never, ExecutionResult], ) -> None: - """Receive all results at once and emit final output.""" + """Receive all results at once and emit Workflow Output.""" await ctx.yield_output(ExecutionResult(results=self._results)) @handler @@ -305,9 +304,7 @@ class CreateConcurrentValidationWorkflowExecutor(Executor): ) collector = CollectorExecutor() - nested_builder = WorkflowBuilder( - start_executor=coordinator, output_executors=[collector] - ) + nested_builder = WorkflowBuilder(start_executor=coordinator, output_from=[collector]) nested_builder.add_edge(coordinator, collector) for worker in workers: nested_builder.add_edge(coordinator, worker)