Create trend report

.github/workflows/python-sample-validation.yml

@@ -57,7 +57,7 @@ jobs:
         if: always()
         with:
           name: validation-report-01-get-started
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents:
     name: Validate 02-agents
@@ -111,7 +111,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-openai:
     name: Validate 02-agents/providers/openai
@@ -150,7 +150,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-openai
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-azure-openai:
     name: Validate 02-agents/providers/azure_openai
@@ -189,7 +189,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-azure-openai
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-azure-ai:
     name: Validate 02-agents/providers/azure_ai
@@ -226,7 +226,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-azure-ai
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-azure-ai-agent:
     name: Validate 02-agents/providers/azure_ai_agent
@@ -263,7 +263,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-azure-ai-agent
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-anthropic:
     name: Validate 02-agents/providers/anthropic
@@ -300,7 +300,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-anthropic
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-github-copilot:
     name: Validate 02-agents/providers/github_copilot
@@ -329,7 +329,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-github-copilot
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-amazon:
     name: Validate 02-agents/providers/amazon
@@ -361,7 +361,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-amazon
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-ollama:
     name: Validate 02-agents/providers/ollama
@@ -393,7 +393,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-ollama
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-foundry-local:
     name: Validate 02-agents/providers/foundry_local
@@ -423,7 +423,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-foundry-local
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-copilotstudio:
     name: Validate 02-agents/providers/copilotstudio
@@ -465,7 +465,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-copilotstudio
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-02-agents-custom:
     name: Validate 02-agents/providers/custom
@@ -494,7 +494,7 @@ jobs:
         if: always()
         with:
           name: validation-report-02-agents-custom
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-03-workflows:
     name: Validate 03-workflows
@@ -539,7 +539,7 @@ jobs:
         if: always()
         with:
           name: validation-report-03-workflows
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-04-hosting:
     name: Validate 04-hosting
@@ -578,7 +578,7 @@ jobs:
         if: always()
         with:
           name: validation-report-04-hosting
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-05-end-to-end:
     name: Validate 05-end-to-end
@@ -622,7 +622,7 @@ jobs:
         if: always()
         with:
           name: validation-report-05-end-to-end
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-autogen-migration:
     name: Validate autogen-migration
@@ -672,7 +672,7 @@ jobs:
         if: always()
         with:
           name: validation-report-autogen-migration
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
 
   validate-semantic-kernel-migration:
     name: Validate semantic-kernel-migration
@@ -733,4 +733,69 @@ jobs:
         if: always()
         with:
           name: validation-report-semantic-kernel-migration
-          path: python/scripts/sample_validation/reports/
+          path: python/samples/sample_validation/reports/
+
+  aggregate-results:
+    name: Aggregate Results
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - validate-01-get-started
+      - validate-02-agents
+      - validate-02-agents-openai
+      - validate-02-agents-azure-openai
+      - validate-02-agents-azure-ai
+      - validate-02-agents-azure-ai-agent
+      - validate-02-agents-anthropic
+      - validate-02-agents-github-copilot
+      - validate-02-agents-amazon
+      - validate-02-agents-ollama
+      - validate-02-agents-foundry-local
+      - validate-02-agents-copilotstudio
+      - validate-02-agents-custom
+      - validate-03-workflows
+      - validate-04-hosting
+      - validate-05-end-to-end
+      - validate-autogen-migration
+      - validate-semantic-kernel-migration
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Download all validation reports
+        uses: actions/download-artifact@v7
+        with:
+          pattern: validation-report-*
+          path: reports/
+          merge-multiple: true
+
+      - name: Restore validation history
+        id: cache-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: validation-history/
+          key: validation-history-${{ github.run_id }}
+          restore-keys: |
+            validation-history-
+
+      - name: Aggregate results and generate trend report
+        run: |
+          python3 python/scripts/sample_validation/aggregate.py \
+            reports/ \
+            validation-history/history.json \
+            trend-report.md
+
+      - name: Write trend report to job summary
+        run: cat trend-report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Save validation history
+        uses: actions/cache/save@v4
+        with:
+          path: validation-history/
+          key: validation-history-${{ github.run_id }}
+
+      - name: Upload trend report
+        uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: validation-trend-report
+          path: trend-report.md

python/scripts/sample_validation/aggregate.py

@@ -0,0 +1,215 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+"""Aggregate validation reports across runs and produce a trend report.
+
+Reads JSON reports from individual validation jobs, combines them with
+cached history from previous runs, and produces a markdown trend report
+showing per-sample status over the last 5 runs.
+
+Usage:
+    python aggregate.py <reports-dir> <history-file> <output-file>
+"""
+
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+MAX_HISTORY = 5
+
+STATUS_EMOJI = {
+    "success": "✅",
+    "failure": "❌",
+    "missing_setup": "⚠️",
+}
+
+
+def load_current_run(reports_dir: Path) -> dict[str, Any]:
+    """Load all JSON report files from the current run and merge them."""
+    combined_results: dict[str, str] = {}
+    total = success = failure = missing = 0
+
+    json_files = sorted(reports_dir.glob("*.json"))
+    if not json_files:
+        print(f"Warning: No JSON report files found in {reports_dir}")
+        return {
+            "timestamp": datetime.now().isoformat(),
+            "summary": {
+                "total_samples": 0,
+                "success_count": 0,
+                "failure_count": 0,
+                "missing_setup_count": 0,
+            },
+            "results": {},
+        }
+
+    for json_file in json_files:
+        print(f"  Loading report: {json_file.name}")
+        with open(json_file, encoding="utf-8") as f:
+            report = json.load(f)
+        for result in report["results"]:
+            combined_results[result["path"]] = result["status"]
+        summary = report["summary"]
+        total += summary["total_samples"]
+        success += summary["success_count"]
+        failure += summary["failure_count"]
+        missing += summary["missing_setup_count"]
+
+    return {
+        "timestamp": datetime.now().isoformat(),
+        "summary": {
+            "total_samples": total,
+            "success_count": success,
+            "failure_count": failure,
+            "missing_setup_count": missing,
+        },
+        "results": combined_results,
+    }
+
+
+def load_history(history_path: Path) -> list[dict[str, Any]]:
+    """Load previous run history from cache."""
+    if history_path.exists():
+        with open(history_path, encoding="utf-8") as f:
+            data = json.load(f)
+        runs = data.get("runs", [])
+        print(f"  Loaded {len(runs)} previous run(s) from history")
+        return runs
+    print("  No previous history found")
+    return []
+
+
+def save_history(history_path: Path, runs: list[dict[str, Any]]) -> None:
+    """Save run history, keeping only the last MAX_HISTORY entries."""
+    history_path.parent.mkdir(parents=True, exist_ok=True)
+    trimmed = runs[-MAX_HISTORY:]
+    with open(history_path, "w", encoding="utf-8") as f:
+        json.dump({"runs": trimmed}, f, indent=2)
+    print(f"  Saved {len(trimmed)} run(s) to history")
+
+
+def generate_trend_report(runs: list[dict[str, Any]]) -> str:
+    """Generate a markdown trend report from run history."""
+    lines = [
+        "# Sample Validation Trend Report",
+        "",
+        f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M UTC')}*",
+        "",
+    ]
+
+    # --- Overall status table (most recent first) ---
+    lines.append("## Overall Status (Last 5 Runs)")
+    lines.append("")
+    lines.append("| Run Date | Success | Failure | Missing Setup | Total |")
+    lines.append("|----------|---------|---------|---------------|-------|")
+
+    for run in reversed(runs):
+        s = run["summary"]
+        date = run["timestamp"][:10]
+        lines.append(
+            f"| {date} | {s['success_count']}/{s['total_samples']} "
+            f"| {s['failure_count']}/{s['total_samples']} "
+            f"| {s['missing_setup_count']}/{s['total_samples']} "
+            f"| {s['total_samples']} |"
+        )
+
+    # Pad with N/A rows if fewer than 5 runs
+    for _ in range(MAX_HISTORY - len(runs)):
+        lines.append("| N/A | N/A | N/A | N/A | N/A |")
+
+    lines.append("")
+
+    # --- Per-sample results table ---
+    lines.append("## Per-Sample Results")
+    lines.append("")
+
+    # Collect all sample paths across all runs
+    all_paths: set[str] = set()
+    for run in runs:
+        all_paths.update(run["results"].keys())
+
+    if not all_paths:
+        lines.append("*No sample results available.*")
+        return "\n".join(lines)
+
+    # Build header (most recent run first)
+    header = "| Sample |"
+    separator = "|--------|"
+    for run in reversed(runs):
+        date = run["timestamp"][:10]
+        header += f" {date} |"
+        separator += "------------|"
+    for _ in range(MAX_HISTORY - len(runs)):
+        header += " N/A |"
+        separator += "-----|"
+
+    lines.append(header)
+    lines.append(separator)
+
+    for path in sorted(all_paths):
+        row = f"| `{path}` |"
+        for run in reversed(runs):
+            status = run["results"].get(path, "N/A")
+            emoji = STATUS_EMOJI.get(status, "N/A")
+            row += f" {emoji} |"
+        for _ in range(MAX_HISTORY - len(runs)):
+            row += " N/A |"
+        lines.append(row)
+
+    lines.append("")
+    lines.append("**Legend:** ✅ Success · ❌ Failure · ⚠️ Missing Setup · N/A Not available")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def main() -> int:
+    if len(sys.argv) != 4:
+        print("Usage: python aggregate.py <reports-dir> <history-file> <output-file>")
+        return 1
+
+    reports_dir = Path(sys.argv[1])
+    history_path = Path(sys.argv[2])
+    output_path = Path(sys.argv[3])
+
+    print("Aggregating validation results...")
+
+    # Load current run's reports
+    print(f"\nLoading reports from {reports_dir}:")
+    current_run = load_current_run(reports_dir)
+    s = current_run["summary"]
+    print(
+        f"  Current run: {s['success_count']} success, "
+        f"{s['failure_count']} failure, "
+        f"{s['missing_setup_count']} missing setup "
+        f"(total: {s['total_samples']})"
+    )
+
+    # Load history and append current run
+    print(f"\nLoading history from {history_path}:")
+    runs = load_history(history_path)
+    runs.append(current_run)
+    runs = runs[-MAX_HISTORY:]
+
+    # Save updated history
+    print(f"\nSaving history to {history_path}:")
+    save_history(history_path, runs)
+
+    # Generate trend report
+    print("\nGenerating trend report...")
+    report = generate_trend_report(runs)
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(report, encoding="utf-8")
+    print(f"Trend report written to {output_path}")
+
+    # Also print the report to stdout
+    print("\n" + "=" * 80)
+    print(report)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())