Python: Add missing system instruction attr to invoke_agent span (#4012)

* Add missing sysmte instruction attr to invoke_agent span

* Temp remove azure search gate

* fix pipeline error

python/packages/core/agent_framework/observability.py

@@ -1366,7 +1366,7 @@ class AgentTelemetryLayer:
                     span=span,
                     provider_name=provider_name,
                     messages=messages,
-                    system_instructions=_get_instructions_from_options(options),
+                    system_instructions=_get_instructions_from_options(merged_options),
                 )
 
             span_state = {"closed": False}
@@ -1423,7 +1423,7 @@ class AgentTelemetryLayer:
                         span=span,
                         provider_name=provider_name,
                         messages=messages,
-                        system_instructions=_get_instructions_from_options(options),
+                        system_instructions=_get_instructions_from_options(merged_options),
                     )
                 start_time_stamp = perf_counter()
                 try:

python/packages/core/tests/core/test_observability.py

@@ -2437,3 +2437,163 @@ async def test_tool_arguments_pydantic_preserves_non_ascii_characters(
     # Verify JSON is valid and contains the text
     tool_arguments = json.loads(tool_arguments_json)
     assert tool_arguments["greeting"]["message"] == japanese_text
+
+
+# region Test merged options for instructions
+
+
+@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
+async def test_agent_instructions_from_default_options(
+    mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
+):
+    """Test that instructions from default_options are captured in agent telemetry."""
+    import json
+
+    agent = mock_chat_agent()
+    agent.default_options = {"model_id": "TestModel", "instructions": "Default system instructions."}
+
+    messages = [Message(role="user", text="Test message")]
+    span_exporter.clear()
+    response = await agent.run(messages)
+
+    assert response is not None
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) == 1
+    span = spans[0]
+
+    # Instructions from default_options should be captured
+    assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
+    system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
+    assert len(system_instructions) == 1
+    assert system_instructions[0]["content"] == "Default system instructions."
+
+
+@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
+async def test_agent_instructions_from_options_override(
+    mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
+):
+    """Test that instructions from options are captured when no default_options instructions exist."""
+    import json
+
+    agent = mock_chat_agent()
+    agent.default_options = {"model_id": "TestModel"}  # No default instructions
+
+    messages = [Message(role="user", text="Test message")]
+    span_exporter.clear()
+    response = await agent.run(messages, options={"instructions": "Override instructions."})
+
+    assert response is not None
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) == 1
+    span = spans[0]
+
+    assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
+    system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
+    assert len(system_instructions) == 1
+    assert system_instructions[0]["content"] == "Override instructions."
+
+
+@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
+async def test_agent_instructions_merged_from_default_and_options(
+    mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
+):
+    """Test that instructions from both default_options and options are merged (concatenated)."""
+    import json
+
+    agent = mock_chat_agent()
+    agent.default_options = {"model_id": "TestModel", "instructions": "Default instructions."}
+
+    messages = [Message(role="user", text="Test message")]
+    span_exporter.clear()
+    response = await agent.run(messages, options={"instructions": "Additional instructions."})
+
+    assert response is not None
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) == 1
+    span = spans[0]
+
+    # Merged instructions should contain both default and override, concatenated with newline
+    assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
+    system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
+    assert len(system_instructions) == 1
+    assert "Default instructions." in system_instructions[0]["content"]
+    assert "Additional instructions." in system_instructions[0]["content"]
+
+
+@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
+async def test_agent_streaming_instructions_from_default_options(
+    mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
+):
+    """Test that streaming agent telemetry captures instructions from default_options."""
+    import json
+
+    agent = mock_chat_agent()
+    agent.default_options = {"model_id": "TestModel", "instructions": "Default streaming instructions."}
+
+    messages = [Message(role="user", text="Test message")]
+    span_exporter.clear()
+    updates = []
+    stream = agent.run(messages, stream=True)
+    async for update in stream:
+        updates.append(update)
+    await stream.get_final_response()
+
+    assert len(updates) == 2
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) == 1
+    span = spans[0]
+
+    assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
+    system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
+    assert len(system_instructions) == 1
+    assert system_instructions[0]["content"] == "Default streaming instructions."
+
+
+@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
+async def test_agent_streaming_instructions_merged_from_default_and_options(
+    mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
+):
+    """Test that streaming agent telemetry captures merged instructions from default_options and options."""
+    import json
+
+    agent = mock_chat_agent()
+    agent.default_options = {"model_id": "TestModel", "instructions": "Default instructions."}
+
+    messages = [Message(role="user", text="Test message")]
+    span_exporter.clear()
+    updates = []
+    stream = agent.run(messages, stream=True, options={"instructions": "Stream override."})
+    async for update in stream:
+        updates.append(update)
+    await stream.get_final_response()
+
+    assert len(updates) == 2
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) == 1
+    span = spans[0]
+
+    assert OtelAttr.SYSTEM_INSTRUCTIONS in span.attributes
+    system_instructions = json.loads(span.attributes[OtelAttr.SYSTEM_INSTRUCTIONS])
+    assert len(system_instructions) == 1
+    assert "Default instructions." in system_instructions[0]["content"]
+    assert "Stream override." in system_instructions[0]["content"]
+
+
+@pytest.mark.parametrize("enable_sensitive_data", [True], indirect=True)
+async def test_agent_no_instructions_in_default_or_options(
+    mock_chat_agent, span_exporter: InMemorySpanExporter, enable_sensitive_data
+):
+    """Test that system_instructions is not set when neither default_options nor options have instructions."""
+    agent = mock_chat_agent()
+    agent.default_options = {"model_id": "TestModel"}  # No instructions
+
+    messages = [Message(role="user", text="Test message")]
+    span_exporter.clear()
+    response = await agent.run(messages)
+
+    assert response is not None
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) == 1
+    span = spans[0]
+
+    assert OtelAttr.SYSTEM_INSTRUCTIONS not in span.attributes