Python: Add tool call/result content types and update connectors and samples (#2971)

* Add new AI content types and image tool support

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Add Python content types for tool calls/results and image generation tool support

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Address review feedback for tool content and samples

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Tighten image generation typing and sample tools list

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Align image generation output typing

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Handle MCP naming, image options mapping, and connector tool content

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Allow MCP call in function approval request

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Remove raw image_generation tool remapping

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Restore Anthropic tool_use to function calls unless code execution

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Fix lint issues for hosted file docstring and MCP parsing

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Import ChatResponse types in Anthropic client

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Fix Anthropics citation type imports and MCP typing for handoff/tools

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* Skip lightning tests without agentlightning and fix function call import

Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>

* fix lint on lab package

* rebuilt anthropic parsing

* redid anthropic parsing

* typo

* updated parsing and added missing docstrings

* fix tests

* mypy fixes

* second mypy fix

* add new class to other samples

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: eavanvalkenburg <13749212+eavanvalkenburg@users.noreply.github.com>
Co-authored-by: eavanvalkenburg <github@vanvalkenburg.eu>

python/samples/getting_started/agents/azure_ai/azure_ai_with_image_generation.py

@@ -3,7 +3,7 @@ import asyncio
 from pathlib import Path
 
 import aiofiles
-from agent_framework import DataContent
+from agent_framework import DataContent, HostedImageGenerationTool
 from agent_framework.azure import AzureAIClient
 from azure.identity.aio import AzureCliCredential
 
@@ -29,12 +29,13 @@ async def main() -> None:
             name="ImageGenAgent",
             instructions="Generate images based on user requirements.",
             tools=[
-                {
-                    "type": "image_generation",
-                    "model": "gpt-image-1-mini",
-                    "quality": "low",
-                    "size": "1024x1024",
-                }
+                HostedImageGenerationTool(
+                    options={
+                        "model": "gpt-image-1-mini",
+                        "quality": "low",
+                        "size": "1024x1024",
+                    }
+                )
             ],
         ) as agent,
     ):

python/samples/getting_started/agents/openai/openai_responses_client_image_generation.py

@@ -3,7 +3,7 @@
 import asyncio
 import base64
 
-from agent_framework import DataContent, UriContent
+from agent_framework import DataContent, HostedImageGenerationTool, ImageGenerationToolResultContent, UriContent
 from agent_framework.openai import OpenAIResponsesClient
 
 """
@@ -51,14 +51,12 @@ async def main() -> None:
     agent = OpenAIResponsesClient().create_agent(
         instructions="You are a helpful AI that can generate images.",
         tools=[
-            {
-                "type": "image_generation",
-                # Core parameters
-                "size": "1024x1024",
-                "background": "transparent",
-                "quality": "low",
-                "format": "webp",
-            }
+            HostedImageGenerationTool(
+                options={
+                    "size": "1024x1024",
+                    "output_format": "webp",
+                }
+            )
         ],
     )
 
@@ -72,9 +70,11 @@ async def main() -> None:
     # Show information about the generated image
     for message in result.messages:
         for content in message.contents:
-            if isinstance(content, (DataContent, UriContent)) and content.uri:
-                show_image_info(content.uri)
-                break
+            if isinstance(content, ImageGenerationToolResultContent) and content.outputs:
+                for output in content.outputs:
+                    if isinstance(output, (DataContent, UriContent)) and output.uri:
+                        show_image_info(output.uri)
+                        break
 
 
 if __name__ == "__main__":

python/samples/getting_started/agents/openai/openai_responses_client_streaming_image_generation.py

@@ -4,7 +4,7 @@ import asyncio
 import base64
 
 import anyio
-from agent_framework import DataContent
+from agent_framework import DataContent, HostedImageGenerationTool
 from agent_framework.openai import OpenAIResponsesClient
 
 """OpenAI Responses Client Streaming Image Generation Example
@@ -45,12 +45,13 @@ async def main():
     agent = OpenAIResponsesClient().create_agent(
         instructions="You are a helpful agent that can generate images.",
         tools=[
-            {
-                "type": "image_generation",
-                "size": "1024x1024",
-                "quality": "high",
-                "partial_images": 3,
-            }
+            HostedImageGenerationTool(
+                options={
+                    "size": "1024x1024",
+                    "quality": "high",
+                    "partial_images": 3,
+                }
+            )
         ],
     )
 

python/samples/getting_started/agents/openai/openai_responses_client_with_code_interpreter.py

@@ -2,10 +2,14 @@
 
 import asyncio
 
-from agent_framework import ChatAgent, ChatResponse, HostedCodeInterpreterTool
+from agent_framework import (
+    ChatAgent,
+    CodeInterpreterToolCallContent,
+    CodeInterpreterToolResultContent,
+    HostedCodeInterpreterTool,
+    TextContent,
+)
 from agent_framework.openai import OpenAIResponsesClient
-from openai.types.responses.response import Response as OpenAIResponse
-from openai.types.responses.response_code_interpreter_tool_call import ResponseCodeInterpreterToolCall
 
 """
 OpenAI Responses Client with Code Interpreter Example
@@ -30,15 +34,20 @@ async def main() -> None:
     result = await agent.run(query)
     print(f"Result: {result}\n")
 
-    if (
-        isinstance(result.raw_representation, ChatResponse)
-        and isinstance(result.raw_representation.raw_representation, OpenAIResponse)
-        and len(result.raw_representation.raw_representation.output) > 0
-        and isinstance(result.raw_representation.raw_representation.output[0], ResponseCodeInterpreterToolCall)
-    ):
-        generated_code = result.raw_representation.raw_representation.output[0].code
-
-        print(f"Generated code:\n{generated_code}")
+    for message in result.messages:
+        code_blocks = [c for c in message.contents if isinstance(c, CodeInterpreterToolCallContent)]
+        outputs = [c for c in message.contents if isinstance(c, CodeInterpreterToolResultContent)]
+        if code_blocks:
+            code_inputs = code_blocks[0].inputs or []
+            for content in code_inputs:
+                if isinstance(content, TextContent):
+                    print(f"Generated code:\n{content.text}")
+                    break
+        if outputs:
+            print("Execution outputs:")
+            for out in outputs[0].outputs or []:
+                if isinstance(out, TextContent):
+                    print(out.text)
 
 
 if __name__ == "__main__":