mirror of
https://github.com/microsoft/agent-framework.git
synced 2026-06-16 21:04:09 +08:00
Fix integration test worker crashes in Azure Functions on Py3.13 (#4260)
* Initial plan * Fix integration test worker crashes on Python 3.13 Three changes to prevent pytest-xdist workers from crashing during Azure Functions integration tests: 1. Add `start_new_session=True` to subprocess on Linux so signals (e.g. from test-timeout) cannot propagate between the func host and the xdist worker process. 2. Add an overall 100-second budget to the fixture setup loop so the retry logic never exceeds the 120-second test timeout. When pytest-timeout's thread method fires during fixture setup and the thread doesn't respond, it calls os._exit() which kills the xdist worker – this is the root cause of the "Not properly terminated" crashes. 3. Remove the `UV_PYTHON: "3.10"` workaround from both workflow files so integration tests actually run on Python 3.13. Co-authored-by: larohra <41490930+larohra@users.noreply.github.com> * Validate integration tests on Python 3.13 Co-authored-by: larohra <41490930+larohra@users.noreply.github.com> * Revert unintentional uv.lock dependency bumps Co-authored-by: larohra <41490930+larohra@users.noreply.github.com> * Use time.monotonic() instead of time.time() for fixture budget timing Addresses review feedback: monotonic clock is immune to NTP/clock adjustments that could skew the budget enforcement. Co-authored-by: larohra <41490930+larohra@users.noreply.github.com> * Fix func worker segfault on Python 3.13 by redirecting worker to Python 3.12 The Azure Functions Python worker crashes with SIGSEGV (exit code 139) on Python 3.13 due to protobuf C extension (google._upb) compatibility issues. When the test runner uses Python >=3.13, the conftest now automatically finds a compatible Python 3.10-3.12 and sets languageWorkers__python__defaultExecutablePath so the func host uses it for the worker process. The CI setup action also ensures Python 3.12 is available on the runner, falling back to uv python install if the system doesn't have it. Co-authored-by: larohra <41490930+larohra@users.noreply.github.com> * Address code review: add path validation, clarify version range and config key format Co-authored-by: larohra <41490930+larohra@users.noreply.github.com> * Run func worker natively on Python 3.13 by disabling dependency isolation Replace the Python 3.12 redirect workaround with the proper fix: set PYTHON_ISOLATE_WORKER_DEPENDENCIES=0 on Python >=3.13. The segfault (exit code 139) is caused by the Azure Functions worker's module isolation mechanism conflicting with protobuf's C extensions (google._upb) on Python 3.13. Disabling isolation lets the worker load dependencies from the app's own environment, which avoids the crash while keeping everything running on Python 3.13. See: https://github.com/Azure/azure-functions-python-worker/issues/1797 Co-authored-by: larohra <41490930+larohra@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: larohra <41490930+larohra@users.noreply.github.com> Co-authored-by: Laveesh Rohra <larohra@microsoft.com>
This commit is contained in:
committed by
GitHub
Unverified
parent
8b0db48d33
commit
b59a854fcd
@@ -365,6 +365,14 @@ def _start_function_app(sample_path: Path, port: int) -> subprocess.Popen[Any]:
|
||||
# use the task hub name to separate orchestration state.
|
||||
env["TASKHUB_NAME"] = f"test{uuid.uuid4().hex[:8]}"
|
||||
|
||||
# The Azure Functions Python worker's dependency isolation mechanism crashes
|
||||
# on Python 3.13 with a SIGSEGV in the protobuf C extension (google._upb).
|
||||
# Disabling isolation lets the worker load dependencies from the app's own
|
||||
# environment, which avoids the crash.
|
||||
# See: https://github.com/Azure/azure-functions-python-worker/issues/1797
|
||||
if sys.version_info >= (3, 13):
|
||||
env.setdefault("PYTHON_ISOLATE_WORKER_DEPENDENCIES", "0")
|
||||
|
||||
# On Windows, use CREATE_NEW_PROCESS_GROUP to allow proper termination
|
||||
# shell=True only on Windows to handle PATH resolution
|
||||
if sys.platform == "win32":
|
||||
@@ -375,8 +383,15 @@ def _start_function_app(sample_path: Path, port: int) -> subprocess.Popen[Any]:
|
||||
shell=True,
|
||||
env=env,
|
||||
)
|
||||
# On Unix, don't use shell=True to avoid shell wrapper issues
|
||||
return subprocess.Popen(["func", "start", "--port", str(port)], cwd=str(sample_path), env=env)
|
||||
# On Unix, use start_new_session=True to isolate the process group from the
|
||||
# pytest-xdist worker. Without this, signals (e.g. from test-timeout) can
|
||||
# propagate to the func host and vice-versa, potentially killing the worker.
|
||||
return subprocess.Popen(
|
||||
["func", "start", "--port", str(port)],
|
||||
cwd=str(sample_path),
|
||||
env=env,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_function_app_ready(func_process: subprocess.Popen[Any], port: int, max_wait: int = 60) -> None:
|
||||
@@ -533,18 +548,33 @@ def function_app_for_test(request: pytest.FixtureRequest) -> Iterator[dict[str,
|
||||
_load_and_validate_env(sample_path)
|
||||
|
||||
max_attempts = 3
|
||||
# The overall budget MUST be shorter than the pytest-timeout value
|
||||
# (--timeout=120 by default) so that the fixture finishes cleanly instead
|
||||
# of being killed by os._exit() which crashes the xdist worker.
|
||||
overall_budget = 100 # seconds – leaves headroom below the 120 s test timeout
|
||||
last_error: Exception | None = None
|
||||
func_process: subprocess.Popen[Any] | None = None
|
||||
base_url = ""
|
||||
port = 0
|
||||
overall_start = time.monotonic()
|
||||
attempts_made = 0
|
||||
|
||||
for _ in range(max_attempts):
|
||||
remaining = overall_budget - (time.monotonic() - overall_start)
|
||||
if remaining < 10:
|
||||
# Not enough time for another attempt; bail out.
|
||||
break
|
||||
|
||||
attempts_made += 1
|
||||
port = _find_available_port()
|
||||
base_url = _build_base_url(port)
|
||||
func_process = _start_function_app(sample_path, port)
|
||||
|
||||
try:
|
||||
_wait_for_function_app_ready(func_process, port)
|
||||
# Cap each attempt's wait to the remaining budget minus a small
|
||||
# buffer for cleanup.
|
||||
per_attempt_wait = min(60, int(remaining) - 5)
|
||||
_wait_for_function_app_ready(func_process, port, max_wait=max(per_attempt_wait, 10))
|
||||
last_error = None
|
||||
break
|
||||
except FunctionAppStartupError as exc:
|
||||
@@ -553,7 +583,8 @@ def function_app_for_test(request: pytest.FixtureRequest) -> Iterator[dict[str,
|
||||
func_process = None
|
||||
|
||||
if func_process is None:
|
||||
error_message = f"Function app failed to start after {max_attempts} attempt(s)."
|
||||
elapsed = int(time.monotonic() - overall_start)
|
||||
error_message = f"Function app failed to start after {attempts_made} attempt(s) ({elapsed}s elapsed)."
|
||||
if last_error is not None:
|
||||
error_message += f" Last error: {last_error}"
|
||||
pytest.fail(error_message)
|
||||
|
||||
Reference in New Issue
Block a user