From 4b9856e66fe527417e71ff84886057a2fb3a9557 Mon Sep 17 00:00:00 2001 From: Eduard van Valkenburg Date: Wed, 1 Apr 2026 15:35:56 +0200 Subject: [PATCH] Python: updated azure ai inference sample (#5028) * updated azure ai inference sample * openai multimodel fix * update language --- .../core/agent_framework/azure/__init__.py | 4 ++++ .../core/agent_framework/azure/__init__.pyi | 8 +++++++ .../azure_ai_inference_embeddings.py | 22 ++++++++++++------- .../openai_chat_multimodal.py | 15 +++++++++---- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/python/packages/core/agent_framework/azure/__init__.py b/python/packages/core/agent_framework/azure/__init__.py index 9cd961c921..27a9dc7e3a 100644 --- a/python/packages/core/agent_framework/azure/__init__.py +++ b/python/packages/core/agent_framework/azure/__init__.py @@ -15,6 +15,10 @@ _IMPORTS: dict[str, tuple[str, str]] = { "AzureAISearchContextProvider": ("agent_framework_azure_ai_search", "agent-framework-azure-ai-search"), "AzureAISearchSettings": ("agent_framework_azure_ai_search", "agent-framework-azure-ai-search"), "AzureAISettings": ("agent_framework_azure_ai", "agent-framework-azure-ai"), + "AzureAIInferenceEmbeddingClient": ("agent_framework_azure_ai", "agent-framework-azure-ai"), + "AzureAIInferenceEmbeddingOptions": ("agent_framework_azure_ai", "agent-framework-azure-ai"), + "AzureAIInferenceEmbeddingSettings": ("agent_framework_azure_ai", "agent-framework-azure-ai"), + "RawAzureAIInferenceEmbeddingClient": ("agent_framework_azure_ai", "agent-framework-azure-ai"), "AzureCredentialTypes": ("agent_framework_azure_ai", "agent-framework-azure-ai"), "AzureTokenProvider": ("agent_framework_azure_ai", "agent-framework-azure-ai"), "DurableAIAgent": ("agent_framework_durabletask", "agent-framework-durabletask"), diff --git a/python/packages/core/agent_framework/azure/__init__.pyi b/python/packages/core/agent_framework/azure/__init__.pyi index 13694a1017..52d32bb146 100644 --- a/python/packages/core/agent_framework/azure/__init__.pyi +++ b/python/packages/core/agent_framework/azure/__init__.pyi @@ -4,9 +4,13 @@ # Install the relevant packages for full type support. from agent_framework_azure_ai import ( + AzureAIInferenceEmbeddingClient, + AzureAIInferenceEmbeddingOptions, + AzureAIInferenceEmbeddingSettings, AzureAISettings, AzureCredentialTypes, AzureTokenProvider, + RawAzureAIInferenceEmbeddingClient, ) from agent_framework_azure_ai_search import ( AzureAISearchContextProvider, @@ -26,6 +30,9 @@ __all__ = [ "AgentCallbackContext", "AgentFunctionApp", "AgentResponseCallbackProtocol", + "AzureAIInferenceEmbeddingClient", + "AzureAIInferenceEmbeddingOptions", + "AzureAIInferenceEmbeddingSettings", "AzureAISearchContextProvider", "AzureAISearchSettings", "AzureAISettings", @@ -35,4 +42,5 @@ __all__ = [ "DurableAIAgentClient", "DurableAIAgentOrchestrationContext", "DurableAIAgentWorker", + "RawAzureAIInferenceEmbeddingClient", ] diff --git a/python/samples/02-agents/embeddings/azure_ai_inference_embeddings.py b/python/samples/02-agents/embeddings/azure_ai_inference_embeddings.py index 70d60983e9..e806166f16 100644 --- a/python/samples/02-agents/embeddings/azure_ai_inference_embeddings.py +++ b/python/samples/02-agents/embeddings/azure_ai_inference_embeddings.py @@ -12,7 +12,7 @@ import asyncio import pathlib from agent_framework import Content -from agent_framework_azure_ai import AzureAIInferenceEmbeddingClient +from agent_framework.azure import AzureAIInferenceEmbeddingClient from dotenv import load_dotenv load_dotenv() @@ -24,8 +24,12 @@ Azure AI Inference embedding client with the Cohere-embed-v3-english model. Images are passed as ``Content`` objects created with ``Content.from_data()``. Prerequisites: - Set the following environment variables or add them to a .env file: - - AZURE_AI_INFERENCE_ENDPOINT: Your Azure AI model inference endpoint URL + Deploy an embedding model in Azure AI Inference that supports image inputs, such as Cohere-embed-v3-english. + + The details page for that model, has a target URI and a Key, which should be set in environment variables or a .env + file as follows, the target URI should append the `/models` path: + - AZURE_AI_INFERENCE_ENDPOINT: Your Azure AI model inference endpoint URL, for instance: + https://.azure-api.net//models - AZURE_AI_INFERENCE_API_KEY: Your API key - AZURE_AI_INFERENCE_EMBEDDING_MODEL_ID: The text embedding model name (e.g. "text-embedding-3-small") @@ -73,15 +77,17 @@ if __name__ == "__main__": """ -Sample output (using Cohere-embed-v3-english): +Sample output (using deployment: Cohere-embed-v3-english, which is Cohere's "embed-english-v3.0-image" model): Image embedding dimensions: 1024 -First 5 values: [0.023, -0.045, 0.067, -0.089, 0.011] -Model: Cohere-embed-v3-english -Usage: {'prompt_tokens': 1, 'total_tokens': 1} +First 5 values: [0.029159546, -0.007926941, -0.0032978058, -0.0030403137, -0.012786865] +Model: embed-english-v3.0-image +Usage: {'input_token_count': 1000, 'output_token_count': 0} -Image+text (separate) results: Text embedding dimensions: 1536 +First 5 values: [-0.019439403, 0.015791258, 0.012358093, 0.0028533707, -0.01649483] Image embedding dimensions: 1024 +First 5 values: [0.029159546, -0.007926941, -0.0032978058, -0.0030403137, -0.012786865] Document embedding dimensions: 1024 +First 5 values: [0.029159546, -0.007926941, -0.0032978058, -0.0030403137, -0.012786865] """ diff --git a/python/samples/02-agents/multimodal_input/openai_chat_multimodal.py b/python/samples/02-agents/multimodal_input/openai_chat_multimodal.py index 2879588a41..3fc8bae84d 100644 --- a/python/samples/02-agents/multimodal_input/openai_chat_multimodal.py +++ b/python/samples/02-agents/multimodal_input/openai_chat_multimodal.py @@ -6,7 +6,7 @@ import struct from pathlib import Path from agent_framework import Content, Message -from agent_framework.foundry import FoundryChatClient +from agent_framework.openai import OpenAIChatClient, OpenAIChatCompletionClient from dotenv import load_dotenv # Load environment variables from .env file @@ -14,6 +14,13 @@ load_dotenv() ASSETS_DIR = Path(__file__).resolve().parents[2] / "shared" / "sample_assets" +""" +Leverage multimodel capabilities of different models. + +Uses the OpenAIChatClient and OpenAIChatCompletionClient to demonstrate multimodal input handling with the gpt-4o and gpt-4o-audio-preview models, respectively. The sample includes demonstrations for image, audio, and PDF inputs, showcasing how to create appropriate Content objects and send them in messages to the chat clients. + +""" + def load_sample_pdf() -> bytes: """Read the bundled sample PDF for tests.""" @@ -46,7 +53,7 @@ def create_sample_audio() -> str: async def test_image() -> None: """Test image analysis with OpenAI.""" - client = FoundryChatClient(model="gpt-4o") + client = OpenAIChatClient(model="gpt-4o") image_uri = create_sample_image() message = Message( @@ -63,7 +70,7 @@ async def test_image() -> None: async def test_audio() -> None: """Test audio analysis with OpenAI.""" - client = FoundryChatClient(model="gpt-4o-audio-preview") + client = OpenAIChatCompletionClient(model="gpt-4o-audio-preview-2025-06-03") audio_uri = create_sample_audio() message = Message( @@ -80,7 +87,7 @@ async def test_audio() -> None: async def test_pdf() -> None: """Test PDF document analysis with OpenAI.""" - client = FoundryChatClient(model="gpt-4o") + client = OpenAIChatClient(model="gpt-4o") pdf_bytes = load_sample_pdf() message = Message(