From 101f50134c6c46b8ace2cb7fb0867e417cb9720e Mon Sep 17 00:00:00 2001 From: Giles Odigwe Date: Thu, 23 Apr 2026 14:23:41 -0700 Subject: [PATCH] Enable Ollama integration tests in CI and rename report to Integration Test Report - Install Ollama, cache models (qwen2.5:0.5b + nomic-embed-text), and start server in the Misc integration job for both workflow files - Set OLLAMA_MODEL and OLLAMA_EMBEDDING_MODEL env vars so the 5 Ollama tests are no longer skipped - Rename Flaky Test Report to Integration Test Report throughout (job names, artifact names, cache keys, file names, script titles/docstrings) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/python-integration-tests.yml | 54 +++++++++++++------ .github/workflows/python-merge-tests.yml | 54 +++++++++++++------ python/scripts/flaky_report/__init__.py | 2 +- python/scripts/flaky_report/__main__.py | 8 +-- python/scripts/flaky_report/aggregate.py | 2 +- 5 files changed, 82 insertions(+), 38 deletions(-) diff --git a/.github/workflows/python-integration-tests.yml b/.github/workflows/python-integration-tests.yml index 48d15bceda..a90860cd4d 100644 --- a/.github/workflows/python-integration-tests.yml +++ b/.github/workflows/python-integration-tests.yml @@ -157,6 +157,8 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }} LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }} + OLLAMA_MODEL: qwen2.5:0.5b + OLLAMA_EMBEDDING_MODEL: nomic-embed-text defaults: run: working-directory: python @@ -171,6 +173,26 @@ jobs: with: python-version: ${{ env.UV_PYTHON }} os: ${{ runner.os }} + - name: Install Ollama + run: curl -fsSL https://ollama.com/install.sh | sh + working-directory: . + - name: Cache Ollama models + uses: actions/cache@v4 + with: + path: ~/.ollama/models + key: ollama-models-qwen2.5-0.5b-nomic-embed-text-v1 + - name: Start Ollama and pull models + run: | + ollama serve & + for i in $(seq 1 30); do + if curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then + break + fi + sleep 1 + done + ollama pull qwen2.5:0.5b + ollama pull nomic-embed-text + working-directory: . - name: Start local MCP server id: local-mcp uses: ./.github/actions/setup-local-mcp-server @@ -388,9 +410,9 @@ jobs: path: ./python/pytest.xml if-no-files-found: ignore - # Flaky test trend report (aggregates per-job JUnit XML results) - python-flaky-test-report: - name: Flaky Test Report + # Integration test trend report (aggregates per-job JUnit XML results) + python-integration-test-report: + name: Integration Test Report if: > always() && (contains(join(needs.*.result, ','), 'success') || @@ -423,36 +445,36 @@ jobs: with: pattern: test-results-* path: test-results/ - - name: Restore flaky report history cache + - name: Restore report history cache uses: actions/cache/restore@v4 with: - path: python/flaky-report-history.json - key: flaky-report-history-integration-${{ github.run_id }} + path: python/integration-report-history.json + key: integration-report-history-integration-${{ github.run_id }} restore-keys: | - flaky-report-history-integration- + integration-report-history-integration- - name: Generate trend report run: > uv run python scripts/flaky_report/aggregate.py ../test-results/ - flaky-report-history.json - flaky-test-report.md + integration-report-history.json + integration-test-report.md - name: Post to Job Summary if: always() - run: cat flaky-test-report.md >> $GITHUB_STEP_SUMMARY - - name: Save flaky report history cache + run: cat integration-test-report.md >> $GITHUB_STEP_SUMMARY + - name: Save report history cache if: always() uses: actions/cache/save@v4 with: - path: python/flaky-report-history.json - key: flaky-report-history-integration-${{ github.run_id }} + path: python/integration-report-history.json + key: integration-report-history-integration-${{ github.run_id }} - name: Upload unified trend report if: always() uses: actions/upload-artifact@v7 with: - name: flaky-test-report + name: integration-test-report path: | - python/flaky-test-report.md - python/flaky-report-history.json + python/integration-test-report.md + python/integration-report-history.json python-integration-tests-check: if: always() diff --git a/.github/workflows/python-merge-tests.yml b/.github/workflows/python-merge-tests.yml index 843253e788..da3c377f53 100644 --- a/.github/workflows/python-merge-tests.yml +++ b/.github/workflows/python-merge-tests.yml @@ -275,6 +275,8 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} ANTHROPIC_CHAT_MODEL: ${{ vars.ANTHROPIC_CHAT_MODEL_ID }} LOCAL_MCP_URL: ${{ vars.LOCAL_MCP__URL }} + OLLAMA_MODEL: qwen2.5:0.5b + OLLAMA_EMBEDDING_MODEL: nomic-embed-text defaults: run: working-directory: python @@ -286,6 +288,26 @@ jobs: with: python-version: ${{ env.UV_PYTHON }} os: ${{ runner.os }} + - name: Install Ollama + run: curl -fsSL https://ollama.com/install.sh | sh + working-directory: . + - name: Cache Ollama models + uses: actions/cache@v4 + with: + path: ~/.ollama/models + key: ollama-models-qwen2.5-0.5b-nomic-embed-text-v1 + - name: Start Ollama and pull models + run: | + ollama serve & + for i in $(seq 1 30); do + if curl -sf http://localhost:11434/api/tags > /dev/null 2>&1; then + break + fi + sleep 1 + done + ollama pull qwen2.5:0.5b + ollama pull nomic-embed-text + working-directory: . - name: Start local MCP server id: local-mcp uses: ./.github/actions/setup-local-mcp-server @@ -555,9 +577,9 @@ jobs: path: ./python/pytest.xml if-no-files-found: ignore - # Flaky test trend report (aggregates per-job JUnit XML results) - python-flaky-test-report: - name: Flaky Test Report + # Integration test trend report (aggregates per-job JUnit XML results) + python-integration-test-report: + name: Integration Test Report if: > always() && (contains(join(needs.*.result, ','), 'success') || @@ -587,36 +609,36 @@ jobs: with: pattern: test-results-* path: test-results/ - - name: Restore flaky report history cache + - name: Restore report history cache uses: actions/cache/restore@v4 with: - path: python/flaky-report-history.json - key: flaky-report-history-merge-${{ github.run_id }} + path: python/integration-report-history.json + key: integration-report-history-merge-${{ github.run_id }} restore-keys: | - flaky-report-history-merge- + integration-report-history-merge- - name: Generate trend report run: > uv run python scripts/flaky_report/aggregate.py ../test-results/ - flaky-report-history.json - flaky-test-report.md + integration-report-history.json + integration-test-report.md - name: Post to Job Summary if: always() - run: cat flaky-test-report.md >> $GITHUB_STEP_SUMMARY - - name: Save flaky report history cache + run: cat integration-test-report.md >> $GITHUB_STEP_SUMMARY + - name: Save report history cache if: always() uses: actions/cache/save@v4 with: - path: python/flaky-report-history.json - key: flaky-report-history-merge-${{ github.run_id }} + path: python/integration-report-history.json + key: integration-report-history-merge-${{ github.run_id }} - name: Upload unified trend report if: always() uses: actions/upload-artifact@v7 with: - name: flaky-test-report + name: integration-test-report path: | - python/flaky-test-report.md - python/flaky-report-history.json + python/integration-test-report.md + python/integration-report-history.json python-integration-tests-check: if: always() diff --git a/python/scripts/flaky_report/__init__.py b/python/scripts/flaky_report/__init__.py index e5a0eeb0ca..e3b0cc6de7 100644 --- a/python/scripts/flaky_report/__init__.py +++ b/python/scripts/flaky_report/__init__.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. -"""Flaky test report aggregation and trend generation. +"""Integration test report aggregation and trend generation. Parses JUnit XML (``pytest.xml``) files produced by each CI job, merges them with historical data, and generates a markdown trend report showing diff --git a/python/scripts/flaky_report/__main__.py b/python/scripts/flaky_report/__main__.py index 89969baae6..c944e135f8 100644 --- a/python/scripts/flaky_report/__main__.py +++ b/python/scripts/flaky_report/__main__.py @@ -1,15 +1,15 @@ # Copyright (c) Microsoft. All rights reserved. -"""CLI entry point for the flaky test report tool. +"""CLI entry point for the integration test report tool. Usage: uv run python -m scripts.flaky_report Example (from python/ directory): uv run python -m scripts.flaky_report \\ - ../flaky-reports/ \\ - flaky-report-history.json \\ - flaky-test-report.md + ../test-results/ \\ + integration-report-history.json \\ + integration-test-report.md """ import sys diff --git a/python/scripts/flaky_report/aggregate.py b/python/scripts/flaky_report/aggregate.py index e07a5e136a..e803add730 100644 --- a/python/scripts/flaky_report/aggregate.py +++ b/python/scripts/flaky_report/aggregate.py @@ -247,7 +247,7 @@ def _short_name(nodeid: str) -> str: def generate_trend_report(runs: list[dict[str, Any]]) -> str: """Generate a markdown trend report from run history.""" lines = [ - "# 🔬 Flaky Test Report", + "# 🔬 Integration Test Report", "", f"*Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}*", "",