From 5db350e76dd67e30079eb5aa258d79e7a1b37f8b Mon Sep 17 00:00:00 2001 From: Paul Cayet Date: Fri, 10 Jul 2026 08:59:53 +0000 Subject: [PATCH 1/6] skeleton for the code executor --- .../src/wayflowcore/codeserver/__init__.py | 7 + .../src/wayflowcore/codeserver/backend.py | 100 ++++++++ .../codeserver/backends/__init__.py | 7 + .../codeserver/backends/local_python.py | 73 ++++++ .../codeserver/backends/pythonworker.py | 35 +++ .../src/wayflowcore/codeserver/models.py | 157 ++++++++++++ .../src/wayflowcore/codeserver/service.py | 136 ++++++++++ .../src/wayflowcore/codeserver/sessions.py | 39 +++ .../tools/codeexecutors/__init__.py | 18 ++ .../wayflowcore/tools/codeexecutors/_utils.py | 102 ++++++++ .../tools/codeexecutors/endpointexecutor.py | 29 +++ .../tools/codeexecutors/executor.py | 120 +++++++++ .../codeexecutors/localcontainerexecutor.py | 18 ++ .../tools/codeexecutors/subprocessexecutor.py | 42 ++++ wayflowcore/tests/codeserver/conftest.py | 16 ++ wayflowcore/tests/codeserver/test_models.py | 234 ++++++++++++++++++ .../tests/codeserver/test_service_core.py | 145 +++++++++++ .../tests/codeserver/test_service_failures.py | 104 ++++++++ .../test_service_host_interactions.py | 127 ++++++++++ .../codeserver/test_service_lifecycle.py | 76 ++++++ .../tests/codeserver/test_service_sessions.py | 130 ++++++++++ 21 files changed, 1715 insertions(+) create mode 100644 wayflowcore/src/wayflowcore/codeserver/__init__.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/backend.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/backends/__init__.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/backends/local_python.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/models.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/service.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/sessions.py create mode 100644 wayflowcore/src/wayflowcore/tools/codeexecutors/__init__.py create mode 100644 wayflowcore/src/wayflowcore/tools/codeexecutors/_utils.py create mode 100644 wayflowcore/src/wayflowcore/tools/codeexecutors/endpointexecutor.py create mode 100644 wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py create mode 100644 wayflowcore/src/wayflowcore/tools/codeexecutors/localcontainerexecutor.py create mode 100644 wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py create mode 100644 wayflowcore/tests/codeserver/conftest.py create mode 100644 wayflowcore/tests/codeserver/test_models.py create mode 100644 wayflowcore/tests/codeserver/test_service_core.py create mode 100644 wayflowcore/tests/codeserver/test_service_failures.py create mode 100644 wayflowcore/tests/codeserver/test_service_host_interactions.py create mode 100644 wayflowcore/tests/codeserver/test_service_lifecycle.py create mode 100644 wayflowcore/tests/codeserver/test_service_sessions.py diff --git a/wayflowcore/src/wayflowcore/codeserver/__init__.py b/wayflowcore/src/wayflowcore/codeserver/__init__.py new file mode 100644 index 000000000..62c82b1c8 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/__init__.py @@ -0,0 +1,7 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Code Executor Protocol server models and services.""" diff --git a/wayflowcore/src/wayflowcore/codeserver/backend.py b/wayflowcore/src/wayflowcore/codeserver/backend.py new file mode 100644 index 000000000..705eedaad --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/backend.py @@ -0,0 +1,100 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Backend interfaces for Code Executor Protocol execution.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + +from wayflowcore.codeserver.models import TaskStatus +from wayflowcore.codeserver.sessions import BackendSessionState + + +@dataclass +class CodeExecutorBackend: + """Interface used by :class:`CodeExecutionService` to run code.""" + + execution_timeout_seconds: float = 30.0 + """Maximum wall-clock time allowed for one execution.""" + + def start_script( + self, + source_code: str, + *, + session: BackendSessionState | None = None, + ) -> "BackendExecutionContext": + """Start a script execution. + + Parameters + ---------- + source_code: + Python source code to execute. + session: + Optional retained backend session in which to execute the code. + + Returns + ------- + BackendExecutionContext + Handle for observing and controlling the execution. + """ + raise NotImplementedError + + def start_function( + self, + source_code: str, + function_name: str, + arguments: Mapping[str, Any], + *, + session: BackendSessionState | None = None, + ) -> "BackendExecutionContext": + """Start a named function execution.""" + raise NotImplementedError + + +@dataclass +class BackendExecutionResult: + """Normalized result produced by an execution backend.""" + + status: TaskStatus + stdout: str = "" + stderr: str = "" + structured_content: Any = None + error: str | None = None + metadata: dict[str, Any] | None = None + + +class BackendExecutionContext: + """Handle for one active or completed backend execution.""" + + def get_result(self) -> BackendExecutionResult: + """Return the latest normalized result for this execution.""" + raise NotImplementedError + + def wait(self) -> BackendExecutionResult: + """Wait until the execution reaches a terminal or waiting state.""" + raise NotImplementedError + + def cancel(self) -> BackendExecutionResult: + """Request cancellation and return the resulting backend state.""" + raise NotImplementedError + + +class PythonExecutionPolicy: + """Policy controlling Python source execution inside a worker.""" + + def validate_script(self, source_code: str) -> None: + """Validate source code intended for script execution.""" + raise NotImplementedError + + def validate_function(self, source_code: str, function_name: str) -> None: + """Validate source code and entry point for function execution.""" + raise NotImplementedError + + def build_namespace(self) -> dict[str, Any]: + """Build the initial namespace for a worker execution.""" + raise NotImplementedError diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/__init__.py b/wayflowcore/src/wayflowcore/codeserver/backends/__init__.py new file mode 100644 index 000000000..19149dbb8 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/backends/__init__.py @@ -0,0 +1,7 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Code Executor Protocol backend implementations.""" diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py new file mode 100644 index 000000000..0f8d4c59e --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py @@ -0,0 +1,73 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Local Python execution backend and worker-process handles.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + +from wayflowcore.codeserver.backend import ( + BackendExecutionContext, + BackendExecutionResult, + CodeExecutorBackend, + PythonExecutionPolicy, +) + + +class LocalPythonBackend(CodeExecutorBackend): + """Backend configuration for executing Python locally.""" + + policy: PythonExecutionPolicy | None = None + + def start_script(self, source_code: str, *, session: Any = None) -> BackendExecutionContext: + """Start a Python script in a worker process.""" + raise NotImplementedError + + def start_function( + self, + source_code: str, + function_name: str, + arguments: Mapping[str, Any], + *, + session: Any = None, + ) -> BackendExecutionContext: + """Start a Python function in a worker process.""" + raise NotImplementedError + + +@dataclass +class LocalPythonExecution(BackendExecutionContext): + """Handle for one local Python worker execution.""" + + execution_id: str + result: BackendExecutionResult | None = None + + def get_result(self) -> BackendExecutionResult: + """Return the latest worker result.""" + raise NotImplementedError + + def wait(self) -> BackendExecutionResult: + """Wait for the worker to finish or request host input.""" + raise NotImplementedError + + def cancel(self) -> BackendExecutionResult: + """Terminate the worker and return its cancellation result.""" + raise NotImplementedError + + +@dataclass +class LocalPythonSession: + """Handle for a long-lived local Python session worker.""" + + session_id: str + process: Any = None + lock: Any = field(default=None, repr=False) + + def close(self) -> None: + """Terminate the session worker and release its resources.""" + raise NotImplementedError diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py b/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py new file mode 100644 index 000000000..c860694b4 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py @@ -0,0 +1,35 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Worker process entry point for the local Python backend.""" + +from __future__ import annotations + +from typing import Any + + +def run_script(source_code: str, namespace: dict[str, Any]) -> Any: + """Execute script source code in a worker namespace.""" + raise NotImplementedError + + +def run_function( + source_code: str, + function_name: str, + arguments: dict[str, Any], + namespace: dict[str, Any], +) -> Any: + """Define source code and invoke one named function in a worker namespace.""" + raise NotImplementedError + + +def main() -> None: + """Run the local Python worker command loop.""" + raise NotImplementedError + + +if __name__ == "__main__": + main() diff --git a/wayflowcore/src/wayflowcore/codeserver/models.py b/wayflowcore/src/wayflowcore/codeserver/models.py new file mode 100644 index 000000000..e0e36c434 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/models.py @@ -0,0 +1,157 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Wire models for the Code Executor Protocol.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +TaskStatus: TypeAlias = Literal[ + "working", "input_required", "completed", "failed", "cancelled", "timed_out" +] + +TASK_STATUS_WORKING: Final[Literal["working"]] = "working" +TASK_STATUS_INPUT_REQUIRED: Final[Literal["input_required"]] = "input_required" +TASK_STATUS_COMPLETED: Final[Literal["completed"]] = "completed" +TASK_STATUS_FAILED: Final[Literal["failed"]] = "failed" +TASK_STATUS_CANCELLED: Final[Literal["cancelled"]] = "cancelled" +TASK_STATUS_TIMED_OUT: Final[Literal["timed_out"]] = "timed_out" + + +class CodeExecutorModel(BaseModel): + """Base model with strict protocol fields.""" + + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + +class ScriptInput(CodeExecutorModel): + """Source code executed as a program.""" + + type: Literal["script"] + source_code: str + + +class FunctionInput(CodeExecutorModel): + """Source code containing one named function to invoke.""" + + type: Literal["function"] + source_code: str + function_name: str + arguments: dict[str, Any] + """Named JSON-compatible arguments passed to the function.""" + + +class HostCallbackResponse(CodeExecutorModel): + """A host response supplied to a prior host interaction request.""" + + type: Literal["host_response"] + request_id: str + result: Any = None + """JSON-compatible result returned by the host.""" + + +class TextContent(CodeExecutorModel): + """Text content produced by an execution.""" + + type: Literal["text"] + text: str + stream: Literal["stdout", "stderr"] | None = None + """Originating process stream when the text was captured from execution.""" + + +ExecutionInputItem: TypeAlias = ScriptInput | FunctionInput | HostCallbackResponse +"""Input item accepted by an execution, including a host response.""" + + +class CodeExecutionRequest(CodeExecutorModel): + """Request body for creating a code execution.""" + + language_id: str + input: list[ExecutionInputItem] + + dependencies: list[str] = Field(default_factory=list) + """Dependencies expected to be available in the execution environment.""" + + session_id: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + wait: bool = True + + @model_validator(mode="after") + def _validate_input_count(self) -> CodeExecutionRequest: + if len(self.input) != 1: + raise ValueError("input must contain exactly one executable item") + return self + + +class ExecutionResult(CodeExecutorModel): + """Output item returned by an execution.""" + + type: Literal["output"] + content: list[TextContent] = Field(default_factory=list) + """Content blocks such as captured text output.""" + + structured_content: Any = Field(default=None, alias="structuredContent") + """Optional JSON-compatible structured result, including scalar values and null.""" + + is_error: bool = Field(default=False, alias="isError") + """Whether the output describes an execution error.""" + + +class HostCallbackRequest(CodeExecutorModel): + """A request for the host to perform an interaction on behalf of code.""" + + type: Literal["host_request"] + request_id: str + request_type: Literal["tool_execution"] + name: str + arguments: dict[str, Any] + + +ExecutionOutputItem: TypeAlias = ExecutionResult | HostCallbackRequest +"""Output item returned by an execution: a result or host callback request.""" + + +class ExecutionResponse(CodeExecutorModel): + """Snapshot returned for a code execution.""" + + id: str + object: Literal["response"] + created_at: datetime + status: TaskStatus + completed_at: datetime | None = None + language_id: str + output: list[ExecutionOutputItem] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class HostInteractions(CodeExecutorModel): + """Session configuration for host-initiated interaction requests.""" + + enabled: bool + allowed_request_types: list[Literal["tool_execution"]] = Field(default_factory=list) + + +class CreateSessionRequest(CodeExecutorModel): + """Request body for creating a stateful execution session.""" + + language_id: str + host_interactions: HostInteractions | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + +class SessionSnapshot(CodeExecutorModel): + """Snapshot of a stateful execution session.""" + + id: str + object: Literal["session"] + status: Literal["active", "closing", "closed", "expired"] + language_id: str + host_interactions: HostInteractions | None = None + metadata: dict[str, Any] = Field(default_factory=dict) diff --git a/wayflowcore/src/wayflowcore/codeserver/service.py b/wayflowcore/src/wayflowcore/codeserver/service.py new file mode 100644 index 000000000..726425068 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/service.py @@ -0,0 +1,136 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Service interfaces for Code Executor Protocol execution.""" + +from __future__ import annotations + +from wayflowcore.codeserver.backend import CodeExecutorBackend +from wayflowcore.codeserver.models import ( + CodeExecutionRequest, + CreateSessionRequest, + ExecutionResponse, + SessionSnapshot, +) + + +class CodeExecutionService: + """Coordinates execution requests, storage, sessions, and a backend.""" + + def __init__(self, backend: CodeExecutorBackend) -> None: + """Initialize the service with an execution backend. + + Parameters + ---------- + backend: + Backend responsible for running code. + """ + self.backend = backend + + def execute(self, request: CodeExecutionRequest) -> ExecutionResponse: + """Create an execution and optionally wait for its completion. + + Parameters + ---------- + request: + Execution request to submit. + + Returns + ------- + ExecutionResponse + Current execution snapshot. + """ + raise NotImplementedError + + def create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: + """Create an execution without waiting for completion. + + Parameters + ---------- + request: + Execution request to submit. + + Returns + ------- + ExecutionResponse + Initial execution snapshot. + """ + raise NotImplementedError + + def get_execution(self, execution_id: str) -> ExecutionResponse: + """Return the latest snapshot for an execution. + + Parameters + ---------- + execution_id: + Identifier of the execution to retrieve. + + Returns + ------- + ExecutionResponse + Latest execution snapshot. + """ + raise NotImplementedError + + def cancel_execution(self, execution_id: str) -> ExecutionResponse: + """Request cancellation of an execution. + + Parameters + ---------- + execution_id: + Identifier of the execution to cancel. + + Returns + ------- + ExecutionResponse + Execution snapshot after cancellation is requested. + """ + raise NotImplementedError + + def create_session(self, request: CreateSessionRequest) -> SessionSnapshot: + """Create a stateful execution session. + + Parameters + ---------- + request: + Session configuration, including the session language. + + Returns + ------- + SessionSnapshot + Initial active session snapshot. + """ + raise NotImplementedError + + def get_session(self, session_id: str) -> SessionSnapshot: + """Return the latest snapshot for a session. + + Parameters + ---------- + session_id: + Identifier of the session to retrieve. + + Returns + ------- + SessionSnapshot + Latest session snapshot. + """ + raise NotImplementedError + + def close_session(self, session_id: str) -> SessionSnapshot: + """Close a stateful execution session. + + Parameters + ---------- + session_id: + Identifier of the session to close. + + Returns + ------- + SessionSnapshot + Session snapshot after closure is requested. + """ + raise NotImplementedError diff --git a/wayflowcore/src/wayflowcore/codeserver/sessions.py b/wayflowcore/src/wayflowcore/codeserver/sessions.py new file mode 100644 index 000000000..a1084e6c0 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/sessions.py @@ -0,0 +1,39 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Backend session abstractions for stateful execution.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass +class BackendSessionState: + """Runtime state owned by one stateful backend session.""" + + session_id: str + language_id: str + runtime: Any = None + closed: bool = False + failed: bool = False + + +class SessionRegistry: + """Registry for backend-owned stateful sessions.""" + + def create(self, session_id: str, language_id: str) -> BackendSessionState: + """Create and register a backend session.""" + raise NotImplementedError + + def get(self, session_id: str) -> BackendSessionState: + """Return a registered backend session.""" + raise NotImplementedError + + def close(self, session_id: str) -> None: + """Close and remove a backend session.""" + raise NotImplementedError diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/__init__.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/__init__.py new file mode 100644 index 000000000..29e59ded5 --- /dev/null +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/__init__.py @@ -0,0 +1,18 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from .endpointexecutor import EndpointCodeExecutor +from .executor import CodeExecutor +from .localcontainerexecutor import LocalContainerCodeExecutor +from .subprocessexecutor import SubProcessCodeExecutor, subprocess_execution_enabled + +__all__ = [ + "CodeExecutor", + "EndpointCodeExecutor", + "LocalContainerCodeExecutor", + "SubProcessCodeExecutor", + "subprocess_execution_enabled", +] diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/_utils.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/_utils.py new file mode 100644 index 000000000..27f89ee56 --- /dev/null +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/_utils.py @@ -0,0 +1,102 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + + +from dataclasses import dataclass, field +from typing import Any, Literal + + +class NotGiven: + """Marker for a value that was not supplied.""" + + def __bool__(self) -> Literal[False]: + return False + + +NOT_GIVEN = NotGiven() + + +@dataclass(frozen=True, kw_only=True) +class CodeExecutionStatus: + """Status returned by a code executor.""" + + status: str + """Execution status reported by the server.""" + execution_id: str + """Identifier of the execution reported by the configured server.""" + metadata: dict[str, Any] = field(default_factory=dict) + """Caller data and server-specific execution details.""" + + +@dataclass(frozen=True, kw_only=True) +class CodeExecutionSucceeded(CodeExecutionStatus): + """A completed execution with accepted output.""" + + status: str = "succeeded" + """Execution status reported by the server.""" + + stdout: str = "" + """Captured standard output.""" + stderr: str = "" + """Captured standard error output.""" + result: Any | NotGiven = NOT_GIVEN + """Structured result, or ``NOT_GIVEN`` when no structured result exists.""" + + +@dataclass(frozen=True, kw_only=True) +class CodeExecutionRejected(CodeExecutionStatus): + """An execution rejected before user code started.""" + + status: str = "rejected" + """Execution status reported by the server.""" + + message: str | None = None + """Optional explanation of the rejection.""" + + +@dataclass(frozen=True, kw_only=True) +class CodeExecutionFailed(CodeExecutionStatus): + """An execution that started and then failed.""" + + status: str = "failed" + """Execution status reported by the server.""" + + message: str | None = None + """Optional explanation of the failure.""" + stdout: str = "" + """Captured standard output produced before the failure.""" + stderr: str = "" + """Captured standard error output produced before the failure.""" + + +@dataclass(frozen=True, kw_only=True) +class CodeExecutionTimedOut(CodeExecutionStatus): + """An execution stopped after a timeout.""" + + status: str = "timed_out" + """Execution status reported by the server.""" + + message: str | None = None + """Optional explanation of the timeout.""" + stdout: str = "" + """Captured standard output produced before the timeout.""" + stderr: str = "" + """Captured standard error output produced before the timeout.""" + + +@dataclass(frozen=True, kw_only=True) +class CodeExecutionCancelled(CodeExecutionStatus): + """An execution cancelled before it produced an accepted result.""" + + status: str = "cancelled" + """Execution status reported by the server.""" + + message: str | None = None + """Optional explanation of the cancellation.""" + stdout: str = "" + """Captured standard output produced before cancellation.""" + stderr: str = "" + """Captured standard error output produced before cancellation.""" diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/endpointexecutor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/endpointexecutor.py new file mode 100644 index 000000000..4c337dd24 --- /dev/null +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/endpointexecutor.py @@ -0,0 +1,29 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from dataclasses import dataclass +from typing import Dict, Optional + +from wayflowcore.retrypolicy import RetryPolicy + +from .executor import CodeExecutor + + +@dataclass +class EndpointCodeExecutor(CodeExecutor): + """Run code through a Code Executor endpoint.""" + + url: str + """Code Executor base URL.""" + + headers: dict[str, str] | None = None + """Non-sensitive transport headers.""" + + sensitive_headers: Optional[Dict[str, str]] = None + """Sensitive transport headers.""" + + retry_policy: RetryPolicy | None = None + """Transport retry policy.""" diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py new file mode 100644 index 000000000..9014c0622 --- /dev/null +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py @@ -0,0 +1,120 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Base classes for code executor configurations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping, Sequence + +from wayflowcore.component import Component + +from ._utils import CodeExecutionStatus + + +@dataclass +class CodeExecutor(Component): + """Class to configure a code executor.""" + + timeout_seconds: float + """Maximum wall-clock time allowed for one execution. The default value is ``30``.""" + max_code_chars: int + """Maximum accepted source length in characters. The default value is ``50000``.""" + + def _execute_function( + self, + code: str, + language: str, + function_name: str, + arguments: Mapping[str, Any], + dependencies: Sequence[str] = (), + metadata: Mapping[str, Any] | None = None, + ) -> CodeExecutionStatus: + """Run one named function defined in source code. + + Parameters + ---------- + code: + Source code that defines ``function_name``. + language: + Language identifier understood by the configured server. + function_name: + Name of the function to invoke from ``code``. + arguments: + JSON-compatible named arguments passed to the function. + dependencies: + Dependency declarations required by the source code. + metadata: + Optional JSON-compatible caller data and suggested executor + settings. The executor separates its own settings from caller + correlation data before calling the server. + + Returns + ------- + CodeExecutionStatus + A terminal execution status. + """ + raise NotImplementedError + + async def _execute_function_async( + self, + code: str, + language: str, + function_name: str, + arguments: Mapping[str, Any], + dependencies: Sequence[str] = (), + metadata: Mapping[str, Any] | None = None, + ) -> CodeExecutionStatus: + """Asynchronously run one named function defined in source code.""" + raise NotImplementedError + + def _execute_script( + self, + code: str, + language: str, + dependencies: Sequence[str] = (), + metadata: Mapping[str, Any] | None = None, + ) -> CodeExecutionStatus: + """Run one script. + + Parameters + ---------- + code: + Source code to run as a script. + language: + Language identifier understood by the configured server. + dependencies: + Dependency declarations required by the source code. + metadata: + Optional JSON-compatible caller data and suggested executor + settings. Script execution may use this mapping to carry a raw + response to an earlier host request. + + Returns + ------- + CodeExecutionStatus + A terminal execution status, or ``waiting_for_context`` when the + server asks the host to do work. + """ + raise NotImplementedError + + async def _execute_script_async( + self, + code: str, + language: str, + dependencies: Sequence[str] = (), + metadata: Mapping[str, Any] | None = None, + ) -> CodeExecutionStatus: + """Asynchronously run one script.""" + raise NotImplementedError + + def get_capabilities(self) -> dict[str, Any]: + """Return capability information supplied by the configured server, + which may include the supported languages, supported execution modes, + and any other supported feature. + """ + raise NotImplementedError diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/localcontainerexecutor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/localcontainerexecutor.py new file mode 100644 index 000000000..7e5c3a031 --- /dev/null +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/localcontainerexecutor.py @@ -0,0 +1,18 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + + +from dataclasses import dataclass + +from .executor import CodeExecutor + + +@dataclass +class LocalContainerCodeExecutor(CodeExecutor): + """Run code through a locally started container-backed Code Executor.""" + + image: str + """Container image selected for the execution environment.""" diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py new file mode 100644 index 000000000..033ad2f23 --- /dev/null +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py @@ -0,0 +1,42 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + + +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Iterator + +from .executor import CodeExecutor + + +@dataclass +class SubProcessCodeExecutor(CodeExecutor): + """Run code through a Code Executor subprocess.""" + + +@contextmanager +def subprocess_execution_enabled() -> Iterator[None]: + """ + Temporarily enable subprocess code execution in the current context. + """ + yield + + +def configure_subprocess_executor_runtime( + max_live_processes: int = 4, + max_queued_requests: int = 32, +) -> None: + """ + Configure subprocess executor worker capacity. + + Parameters + ---------- + max_live_processes + Maximum number of worker subprocesses that may run concurrently. + max_queued_requests + Maximum number of execution requests that may wait for worker capacity. + """ + raise NotImplementedError diff --git a/wayflowcore/tests/codeserver/conftest.py b/wayflowcore/tests/codeserver/conftest.py new file mode 100644 index 000000000..43cfbb49b --- /dev/null +++ b/wayflowcore/tests/codeserver/conftest.py @@ -0,0 +1,16 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +import pytest + +from wayflowcore.codeserver.backends.local_python import LocalPythonBackend +from wayflowcore.codeserver.service import CodeExecutionService + + +@pytest.fixture +def python_service() -> CodeExecutionService: + """Provides a service backed by the local Python implementation.""" + return CodeExecutionService(backend=LocalPythonBackend()) diff --git a/wayflowcore/tests/codeserver/test_models.py b/wayflowcore/tests/codeserver/test_models.py new file mode 100644 index 000000000..9eb33fb39 --- /dev/null +++ b/wayflowcore/tests/codeserver/test_models.py @@ -0,0 +1,234 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for the Code Executor Protocol wire models.""" + +import pytest + +from wayflowcore.codeserver.models import ( + CodeExecutionRequest, + ExecutionResponse, + ExecutionResult, + FunctionInput, + HostCallbackRequest, + HostResponse, + ScriptInput, + SessionRequest, + SessionSnapshot, + TextContent, +) + + +def test_script_execution_request_accepts_protocol_shape() -> None: + """Validates a script request.""" + request = CodeExecutionRequest.model_validate( + { + "language_id": "python", + "input": [ + { + "type": "script", + "source_code": "print('hello')", + } + ], + "dependencies": [], + "session_id": "sess_123", + "metadata": {"caller": "test"}, + "wait": True, + } + ) + + assert request.language_id == "python" + assert len(request.input) == 1 + assert request.input[0].type == "script" + assert request.input[0].source_code == "print('hello')" + assert request.dependencies == [] + assert request.session_id == "sess_123" + assert request.metadata == {"caller": "test"} + assert request.wait is True + + +def test_function_execution_request_accepts_protocol_shape() -> None: + """Validates a function request with named JSON arguments.""" + request = CodeExecutionRequest.model_validate( + { + "language_id": "python", + "input": [ + { + "type": "function", + "source_code": "def multiply(a, b): return a * b", + "function_name": "multiply", + "arguments": {"a": 6, "b": 7}, + } + ], + "wait": True, + } + ) + + assert isinstance(request.input[0], FunctionInput) + assert request.input[0].function_name == "multiply" + assert request.input[0].arguments == {"a": 6, "b": 7} + + +def test_execution_request_raises_on_unknown_input_type() -> None: + """Raises when an input item type is not supported by the protocol.""" + with pytest.raises(ValueError): + CodeExecutionRequest.model_validate( + { + "language_id": "python", + "input": [{"type": "unknown", "source_code": "pass"}], + "wait": True, + } + ) + + +def test_execution_request_raises_on_multiple_input_items() -> None: + """Raises when more than one executable input item is supplied in v1.""" + with pytest.raises(ValueError): + CodeExecutionRequest.model_validate( + { + "language_id": "python", + "input": [ + {"type": "script", "source_code": "pass"}, + {"type": "script", "source_code": "pass"}, + ], + "wait": True, + } + ) + + +def test_script_input_raises_on_function_fields() -> None: + """Raises when function-only fields are supplied on a script input item.""" + with pytest.raises(ValueError): + ScriptInput.model_validate( + { + "type": "script", + "source_code": "print('hello')", + "function_name": "main", + } + ) + + +def test_function_input_raises_on_missing_function_name() -> None: + """Raises when a function name is missing from a function input item.""" + with pytest.raises(ValueError): + FunctionInput.model_validate( + { + "type": "function", + "source_code": "def main(): return 1", + "arguments": {}, + } + ) + + +def test_function_input_raises_on_non_object_arguments() -> None: + """Raises when named arguments are not represented by a JSON object.""" + with pytest.raises(ValueError): + FunctionInput.model_validate( + { + "type": "function", + "source_code": "def main(): return 1", + "function_name": "main", + "arguments": [1, 2], # should be {"a": 1, "b": 1} + } + ) + + +@pytest.mark.parametrize("value", [{"answer": 42}, [1, 2], "done", 42, True, None]) +def test_structured_content_accepts_any_json_value(value: object) -> None: + """Accepts any JSON value as structured execution content.""" + output = ExecutionResult.model_validate( + { + "type": "output", + "content": [{"type": "text", "text": "done"}], + "structuredContent": value, + "isError": False, + } + ) + + assert output.structured_content == value + assert isinstance(output.content[0], TextContent) + + +def test_execution_response_accepts_terminal_snapshot() -> None: + """Validates the Pydantic response model returned by the service.""" + response = ExecutionResponse.model_validate( + { + "id": "exec_123", + "object": "response", + "created_at": "2026-06-04T12:00:00Z", + "status": "completed", + "completed_at": "2026-06-04T12:00:02Z", + "language_id": "python", + "output": [ + { + "type": "output", + "content": [{"type": "text", "text": "done\n"}], + } + ], + } + ) + + assert response.id == "exec_123" + assert response.status == "completed" + assert response.output[0].content[0].text == "done\n" + + +def test_host_request_and_host_response_models() -> None: + """Validates the retained host interaction vocabulary.""" + request = HostCallbackRequest.model_validate( + { + "type": "host_request", + "request_id": "req_123", + "request_type": "tool_execution", + "name": "lookup_weather", + "arguments": {"city": "Paris"}, + } + ) + response = HostResponse.model_validate( + { + "type": "host_response", + "request_id": "req_123", + "result": {"temperature": 21}, + } + ) + + assert request.request_type == "tool_execution" + assert request.arguments == {"city": "Paris"} + assert response.request_id == request.request_id + assert response.result == {"temperature": 21} + + +def test_session_request_and_snapshot_models() -> None: + """Validates session creation and lifecycle snapshot models.""" + request = SessionRequest.model_validate( + { + "language_id": "python", + "language_version": "3.12", + "host_interactions": { + "enabled": True, + "allowed_request_types": ["tool_execution"], + }, + "metadata": {"owner": "test"}, + } + ) + snapshot = SessionSnapshot.model_validate( + { + "id": "sess_123", + "object": "session", + "status": "active", + "language_id": "python", + "language_version": "3.12", + "host_interactions": { + "enabled": True, + "allowed_request_types": ["tool_execution"], + }, + "metadata": {"owner": "test"}, + } + ) + + assert request.language_id == snapshot.language_id + assert snapshot.id == "sess_123" + assert snapshot.status == "active" diff --git a/wayflowcore/tests/codeserver/test_service_core.py b/wayflowcore/tests/codeserver/test_service_core.py new file mode 100644 index 000000000..a715f6ae2 --- /dev/null +++ b/wayflowcore/tests/codeserver/test_service_core.py @@ -0,0 +1,145 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for code execution service and backend behavior.""" + +from wayflowcore.codeserver.models import ( + TASK_STATUS_COMPLETED, + CodeExecutionRequest, + ExecutionResponse, + ExecutionResult, + FunctionInput, + ScriptInput, + TextContent, +) +from wayflowcore.codeserver.service import CodeExecutionService + + +def test_service_runs_script_to_completion(python_service: CodeExecutionService) -> None: + """Runs a script through the local Python backend and captures stdout.""" + request = CodeExecutionRequest( + language_id="python", + input=[ + ScriptInput( + type="script", + source_code="print('hello from the backend')", + ) + ], + wait=True, + ) + + response = python_service.execute(request) + + expected_response = ExecutionResponse( + id=response.id, + object="response", + created_at=response.created_at, + status=TASK_STATUS_COMPLETED, + completed_at=response.completed_at, + language_id="python", + output=[ + ExecutionResult( + type="output", + content=[ + TextContent( + type="text", + stream="stdout", + text="hello from the backend\n", + ) + ], + ) + ], + metadata=response.metadata, + ) + + assert response == expected_response + + +def test_service_runs_function_to_completion(python_service: CodeExecutionService) -> None: + """Invokes a named function with JSON arguments and returns structured content.""" + request = CodeExecutionRequest( + language_id="python", + input=[ + FunctionInput( + type="function", + source_code="def multiply(a, b): return a * b", + function_name="multiply", + arguments={"a": 6, "b": 7}, + ) + ], + wait=True, + ) + + response = python_service.execute(request) + + assert response.status == TASK_STATUS_COMPLETED + assert response.output == [ExecutionResult(type="output", structured_content=42)] + + +def test_service_returns_empty_structured_content_for_script_without_result( + python_service: CodeExecutionService, +) -> None: + """Distinguishes no structured result from JSON null.""" + request = CodeExecutionRequest( + language_id="python", + input=[ScriptInput(type="script", source_code="print('done')")], + wait=True, + ) + + response = python_service.execute(request) + + output = response.output[0] + assert isinstance(output, ExecutionResult) + assert "structured_content" not in output.model_fields_set + + +def test_service_preserves_json_null_function_result(python_service: CodeExecutionService) -> None: + """Preserves a function result of None as JSON null.""" + request = CodeExecutionRequest( + language_id="python", + input=[ + FunctionInput( + type="function", + source_code="def produce_null(): return None", + function_name="produce_null", + arguments={}, + ) + ], + wait=True, + ) + + response = service.execute(request) + + output = response.output[0] + assert isinstance(output, ExecutionResult) + assert output.structured_content is None + assert "structured_content" in output.model_fields_set + + +def test_service_captures_stdout_and_stderr(python_service: CodeExecutionService) -> None: + """Captures both standard output and standard error output.""" + request = CodeExecutionRequest( + language_id="python", + input=[ + ScriptInput( + type="script", + source_code=("import sys\n" "print('stdout')\n" "print('stderr', file=sys.stderr)"), + ) + ], + wait=True, + ) + + response = service.execute(request) + + assert response.output == [ + ExecutionResult( + type="output", + content=[ + TextContent(type="text", stream="stdout", text="stdout\n"), + TextContent(type="text", stream="stderr", text="stderr\n"), + ], + ) + ] diff --git a/wayflowcore/tests/codeserver/test_service_failures.py b/wayflowcore/tests/codeserver/test_service_failures.py new file mode 100644 index 000000000..87ff665e2 --- /dev/null +++ b/wayflowcore/tests/codeserver/test_service_failures.py @@ -0,0 +1,104 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for failed code execution service requests.""" + +import pytest + +from wayflowcore.codeserver.models import ( + TASK_STATUS_FAILED, + CodeExecutionRequest, + ExecutionResult, + FunctionInput, + ScriptInput, +) +from wayflowcore.codeserver.service import CodeExecutionService + + +def test_service_raises_on_unsupported_language( + python_service: CodeExecutionService, +) -> None: + """Raises when the requested language is not supported by the backend.""" + request = CodeExecutionRequest( + language_id="ruby", + input=[ScriptInput(type="script", source_code="puts 'hello'")], + wait=True, + ) + + with pytest.raises(ValueError, match="Unsupported language"): + python_service.execute(request) + + +def test_service_returns_failed_response_on_missing_function( + python_service: CodeExecutionService, +) -> None: + """Returns a failed response when the requested function is not defined.""" + request = CodeExecutionRequest( + language_id="python", + input=[ + FunctionInput( + type="function", + source_code="def multiply(a, b): return a * b", + function_name="does_not_exist", + arguments={"a": 1, "b": 2}, + ) + ], + wait=True, + ) + + response = python_service.execute(request) + + assert response.status == TASK_STATUS_FAILED + assert isinstance(response.output[0], ExecutionResult) + assert response.output[0].is_error is True + + +def test_service_returns_failed_response_on_function_exception( + python_service: CodeExecutionService, +) -> None: + """Returns a failed response when the invoked function raises.""" + request = CodeExecutionRequest( + language_id="python", + input=[ + FunctionInput( + type="function", + source_code="def fail(): raise RuntimeError('boom')", + function_name="fail", + arguments={}, + ) + ], + wait=True, + ) + + response = python_service.execute(request) + + assert response.status == TASK_STATUS_FAILED + assert isinstance(response.output[0], ExecutionResult) + assert response.output[0].is_error is True + + +def test_service_returns_failed_response_on_non_json_function_result( + python_service: CodeExecutionService, +) -> None: + """Returns a failed response when a function result is not JSON-compatible.""" + request = CodeExecutionRequest( + language_id="python", + input=[ + FunctionInput( + type="function", + source_code="def create_object(): return object()", + function_name="create_object", + arguments={}, + ) + ], + wait=True, + ) + + response = python_service.execute(request) + + assert response.status == TASK_STATUS_FAILED + assert isinstance(response.output[0], ExecutionResult) + assert response.output[0].is_error is True diff --git a/wayflowcore/tests/codeserver/test_service_host_interactions.py b/wayflowcore/tests/codeserver/test_service_host_interactions.py new file mode 100644 index 000000000..36a6a48a9 --- /dev/null +++ b/wayflowcore/tests/codeserver/test_service_host_interactions.py @@ -0,0 +1,127 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for host-mediated callback interactions.""" + +import pytest + +from wayflowcore.codeserver.models import ( + TASK_STATUS_COMPLETED, + TASK_STATUS_INPUT_REQUIRED, + CodeExecutionRequest, + CreateSessionRequest, + HostCallbackRequest, + HostCallbackResponse, + ScriptInput, +) +from wayflowcore.codeserver.service import CodeExecutionService + + +def test_service_with_python_backend_returns_callback_host_request( + python_service: CodeExecutionService, +) -> None: + """Returns a callback host request when Python code invokes a host function.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + source_code = """ +result = lookup_weather(city="Paris") +print(result) +""" + # The syntax used to create a callback host request may be backend-dependent. + request = CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ScriptInput(type="script", source_code=source_code)], + ) + + response = python_service.execute(request) + + assert response.status == TASK_STATUS_INPUT_REQUIRED + assert isinstance(response.output[0], HostCallbackRequest) + assert response.output[0].request_type == "tool_execution" + assert response.output[0].name == "lookup_weather" + assert response.output[0].arguments == {"city": "Paris"} + + +def test_service_waits_for_callback_host_response( + python_service: CodeExecutionService, +) -> None: + """Leaves an execution waiting while a callback host request is pending.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + request = CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ + ScriptInput( + type="script", + source_code='result = lookup_weather(city="Paris")', + ) + ], + ) + + response = python_service.execute(request) + + assert response.status == TASK_STATUS_INPUT_REQUIRED + assert response.completed_at is None + assert isinstance(response.output[0], HostCallbackRequest) + assert response.output[0].request_id + + +def test_service_resumes_execution_after_callback_host_response( + python_service: CodeExecutionService, +) -> None: + """Resumes a callback execution with a response in the same session.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + request = CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ + ScriptInput( + type="script", + source_code=('result = lookup_weather(city="Paris")\n' "print(result)"), + ) + ], + ) + + waiting_response = python_service.execute(request) + host_request = waiting_response.output[0] + assert isinstance(host_request, HostCallbackRequest) + + continuation_request = CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ + HostCallbackResponse( + type="host_response", + request_id=host_request.request_id, + result={"temperature": 21}, + ) + ], + ) + + response = python_service.execute(continuation_request) + + assert response.status == TASK_STATUS_COMPLETED + + +def test_service_rejects_unknown_callback_host_request_id( + python_service: CodeExecutionService, +) -> None: + """Rejects a callback response without a matching pending host request.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + request = CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ + HostCallbackResponse( + type="host_response", + request_id="req_does_not_exist", + result={"temperature": 21}, + ) + ], + ) + + with pytest.raises(ValueError, match="host request"): + python_service.execute(request) diff --git a/wayflowcore/tests/codeserver/test_service_lifecycle.py b/wayflowcore/tests/codeserver/test_service_lifecycle.py new file mode 100644 index 000000000..09e8e9ba7 --- /dev/null +++ b/wayflowcore/tests/codeserver/test_service_lifecycle.py @@ -0,0 +1,76 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for code execution lifecycle operations.""" + +from wayflowcore.codeserver.models import ( + TASK_STATUS_CANCELLED, + TASK_STATUS_TIMED_OUT, + TASK_STATUS_WORKING, + CodeExecutionRequest, + ScriptInput, +) +from wayflowcore.codeserver.service import CodeExecutionService + + +def test_service_creates_pending_execution_when_wait_is_false( + python_service: CodeExecutionService, +) -> None: + """Creates a pollable execution when waiting is disabled.""" + request = CodeExecutionRequest( + language_id="python", + input=[ScriptInput(type="script", source_code="print('hello')")], + wait=False, + ) + + response = python_service.create_execution(request) + + assert response.id + assert response.status == TASK_STATUS_WORKING + + +def test_service_returns_execution_snapshot( + python_service: CodeExecutionService, +) -> None: + """Returns the latest snapshot for a created execution.""" + request = CodeExecutionRequest( + language_id="python", + input=[ScriptInput(type="script", source_code="print('hello')")], + wait=False, + ) + execution = python_service.create_execution(request) + + snapshot = python_service.get_execution(execution.id) + + assert snapshot.id == execution.id + + +def test_service_cancellation(python_service: CodeExecutionService) -> None: + """Cancels an execution that has not completed.""" + request = CodeExecutionRequest( + language_id="python", + input=[ScriptInput(type="script", source_code="print('hello')")], + wait=False, + ) + execution = python_service.create_execution(request) + + response = python_service.cancel_execution(execution.id) + + assert response.id == execution.id + assert response.status == TASK_STATUS_CANCELLED + + +def test_service_execution_timeout(python_service: CodeExecutionService) -> None: + """Reports a timed-out execution using the backend timeout configuration.""" + request = CodeExecutionRequest( + language_id="python", + input=[ScriptInput(type="script", source_code="while True: pass")], + wait=True, + ) + + response = python_service.execute(request) + + assert response.status == TASK_STATUS_TIMED_OUT diff --git a/wayflowcore/tests/codeserver/test_service_sessions.py b/wayflowcore/tests/codeserver/test_service_sessions.py new file mode 100644 index 000000000..cf52d3fde --- /dev/null +++ b/wayflowcore/tests/codeserver/test_service_sessions.py @@ -0,0 +1,130 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for stateful code execution sessions.""" + +import pytest + +from wayflowcore.codeserver.models import ( + TASK_STATUS_COMPLETED, + CodeExecutionRequest, + CreateSessionRequest, + ExecutionResult, + ScriptInput, + TextContent, +) +from wayflowcore.codeserver.service import CodeExecutionService + + +def test_service_creates_session(python_service: CodeExecutionService) -> None: + """Creates an active Python execution session.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + + assert session.id + assert session.object == "session" + assert session.status == "active" + assert session.language_id == "python" + + +def test_service_runs_code_in_session(python_service: CodeExecutionService) -> None: + """Runs an execution against a created session.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + request = CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ScriptInput(type="script", source_code="print('hello')")], + ) + + response = python_service.execute(request) + + assert response.status == TASK_STATUS_COMPLETED + assert response.language_id == "python" + + +def test_service_reuses_state_within_session(python_service: CodeExecutionService) -> None: + """Reuses runtime state across executions in one session.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + python_service.execute( + CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ScriptInput(type="script", source_code="value = 42")], + ) + ) + + response = python_service.execute( + CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ScriptInput(type="script", source_code="print(value)")], + ) + ) + + assert response.output == [ + ExecutionResult( + type="output", + content=[TextContent(type="text", stream="stdout", text="42\n")], + ) + ] + + +def test_service_does_not_share_state_between_sessions( + python_service: CodeExecutionService, +) -> None: + """Keeps runtime state isolated between sessions.""" + session_a = python_service.create_session(CreateSessionRequest(language_id="python")) + session_b = python_service.create_session(CreateSessionRequest(language_id="python")) + python_service.execute( + CodeExecutionRequest( + language_id="python", + session_id=session_a.id, + input=[ScriptInput(type="script", source_code="value = 42")], + ) + ) + + response = python_service.execute( + CodeExecutionRequest( + language_id="python", + session_id=session_b.id, + input=[ScriptInput(type="script", source_code="print(value)")], + ) + ) + + assert response.status == "failed" + + +def test_service_rejects_language_mismatch_in_session( + python_service: CodeExecutionService, +) -> None: + """Raises when an execution language differs from its session language.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + request = CodeExecutionRequest( + language_id="javascript", + session_id=session.id, + input=[ScriptInput(type="script", source_code="console.log('hello')")], + ) + + with pytest.raises(ValueError, match="language"): + python_service.execute(request) + + +def test_service_closes_session(python_service: CodeExecutionService) -> None: + """Closes a session and prevents further executions in it.""" + session = python_service.create_session(CreateSessionRequest(language_id="python")) + + closed = python_service.close_session(session.id) + + assert closed.id == session.id + assert closed.status == "closed" + + with pytest.raises(ValueError, match="closed"): + python_service.execute( + CodeExecutionRequest( + language_id="python", + session_id=session.id, + input=[ScriptInput(type="script", source_code="print('hello')")], + ) + ) From 448e082114853d49a0560b304afa008a081a268d Mon Sep 17 00:00:00 2001 From: Paul Cayet Date: Thu, 16 Jul 2026 07:27:44 +0000 Subject: [PATCH 2/6] Implement code executor server core --- .../src/wayflowcore/_utils/notgiven.py | 20 + .../src/wayflowcore/codeserver/__init__.py | 6 +- wayflowcore/src/wayflowcore/codeserver/app.py | 72 +++ .../src/wayflowcore/codeserver/backend.py | 84 ++- .../codeserver/backends/__init__.py | 2 +- .../codeserver/backends/local_python.py | 543 +++++++++++++++++- .../codeserver/backends/pythonworker.py | 255 +++++++- .../src/wayflowcore/codeserver/models.py | 36 +- .../src/wayflowcore/codeserver/server.py | 49 ++ .../src/wayflowcore/codeserver/service.py | 180 +++++- .../src/wayflowcore/codeserver/sessions.py | 44 +- .../src/wayflowcore/codeserver/storage.py | 138 +++++ .../wayflowcore/tools/codeexecutors/_utils.py | 12 +- wayflowcore/tests/codeserver/conftest.py | 8 + wayflowcore/tests/codeserver/test_app.py | 79 +++ .../codeserver/test_local_python_backend.py | 312 ++++++++++ wayflowcore/tests/codeserver/test_models.py | 12 +- .../tests/codeserver/test_service_core.py | 4 +- .../test_service_host_interactions.py | 9 +- 19 files changed, 1766 insertions(+), 99 deletions(-) create mode 100644 wayflowcore/src/wayflowcore/_utils/notgiven.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/app.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/server.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/storage.py create mode 100644 wayflowcore/tests/codeserver/test_app.py create mode 100644 wayflowcore/tests/codeserver/test_local_python_backend.py diff --git a/wayflowcore/src/wayflowcore/_utils/notgiven.py b/wayflowcore/src/wayflowcore/_utils/notgiven.py new file mode 100644 index 000000000..b7678642f --- /dev/null +++ b/wayflowcore/src/wayflowcore/_utils/notgiven.py @@ -0,0 +1,20 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Marker types for optional values whose absence is meaningful.""" + +from typing import Literal + + +class NotGiven: + """Marker for a value that was not supplied.""" + + def __bool__(self) -> Literal[False]: + """Evaluate the marker as false.""" + return False + + +NOT_GIVEN = NotGiven() diff --git a/wayflowcore/src/wayflowcore/codeserver/__init__.py b/wayflowcore/src/wayflowcore/codeserver/__init__.py index 62c82b1c8..16112eecc 100644 --- a/wayflowcore/src/wayflowcore/codeserver/__init__.py +++ b/wayflowcore/src/wayflowcore/codeserver/__init__.py @@ -1,7 +1,11 @@ -# Copyright © 2025 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. """Code Executor Protocol server models and services.""" + +from wayflowcore.codeserver.server import CodeExecutorServer + +__all__ = ["CodeExecutorServer"] diff --git a/wayflowcore/src/wayflowcore/codeserver/app.py b/wayflowcore/src/wayflowcore/codeserver/app.py new file mode 100644 index 000000000..f49fbf577 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/app.py @@ -0,0 +1,72 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""FastAPI application for the Code Executor Protocol routes.""" + +from __future__ import annotations + +from fastapi import FastAPI, HTTPException, status + +from wayflowcore.codeserver.models import ( + CodeExecutionRequest, + CodeExecutorCapabilities, + ExecutionResponse, +) +from wayflowcore.codeserver.service import CodeExecutionService + + +def create_code_executor_app( + service: CodeExecutionService, + *, + server_name: str = "wayflow-code-server", + protocol_version: str = "26.1.3", +) -> FastAPI: + """Create a FastAPI application backed by one code execution service.""" + app = FastAPI( + title=server_name, + version=protocol_version, + description="Code Executor Protocol server.", + ) + + @app.get( + "/v1/code-executor", + response_model=CodeExecutorCapabilities, + ) + def get_capabilities() -> CodeExecutorCapabilities: + """Return the server and backend capabilities.""" + return CodeExecutorCapabilities( + protocol_version=protocol_version, + server_name=server_name, + capabilities=service.backend.get_capabilities(), + ) + + @app.post("/v1/executions", response_model=ExecutionResponse) + def create_execution(request: CodeExecutionRequest) -> ExecutionResponse: + """Create an execution and optionally wait for completion.""" + try: + return service.execute(request) + except NotImplementedError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + @app.get("/v1/executions/{execution_id}", response_model=ExecutionResponse) + def get_execution(execution_id: str) -> ExecutionResponse: + """Return the latest execution snapshot.""" + try: + return service.get_execution(execution_id) + except KeyError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + + @app.post("/v1/executions/{execution_id}/cancel", response_model=ExecutionResponse) + def cancel_execution(execution_id: str) -> ExecutionResponse: + """Cancel an active execution.""" + try: + return service.cancel_execution(execution_id) + except KeyError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + + return app diff --git a/wayflowcore/src/wayflowcore/codeserver/backend.py b/wayflowcore/src/wayflowcore/codeserver/backend.py index 705eedaad..3debdac0e 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backend.py +++ b/wayflowcore/src/wayflowcore/codeserver/backend.py @@ -1,4 +1,4 @@ -# Copyright © 2025 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License @@ -8,25 +8,43 @@ from __future__ import annotations +from abc import ABC, abstractmethod +from collections.abc import Mapping from dataclasses import dataclass -from typing import Any, Mapping -from wayflowcore.codeserver.models import TaskStatus -from wayflowcore.codeserver.sessions import BackendSessionState +from wayflowcore._utils.notgiven import NOT_GIVEN, NotGiven +from wayflowcore.codeserver.models import ( + HostCallbackResponse, + HostInteractions, + JsonValue, + TaskStatus, +) +from wayflowcore.codeserver.sessions import BackendSession @dataclass -class CodeExecutorBackend: +class CodeExecutorBackend(ABC): """Interface used by :class:`CodeExecutionService` to run code.""" execution_timeout_seconds: float = 30.0 """Maximum wall-clock time allowed for one execution.""" + @abstractmethod + def get_capabilities(self) -> dict[str, JsonValue]: + """Return capabilities supported by this backend.""" + raise NotImplementedError + + @abstractmethod + def validate_language(self, language_id: str) -> None: + """Validate that this backend supports the requested language.""" + raise NotImplementedError + + @abstractmethod def start_script( self, source_code: str, *, - session: BackendSessionState | None = None, + session: BackendSession | None = None, ) -> "BackendExecutionContext": """Start a script execution. @@ -44,17 +62,53 @@ def start_script( """ raise NotImplementedError + @abstractmethod def start_function( self, source_code: str, function_name: str, - arguments: Mapping[str, Any], + arguments: Mapping[str, JsonValue], *, - session: BackendSessionState | None = None, + session: BackendSession | None = None, ) -> "BackendExecutionContext": """Start a named function execution.""" raise NotImplementedError + @abstractmethod + def create_session( + self, + session_id: str, + language_id: str, + *, + host_interactions: HostInteractions | None = None, + ) -> BackendSession: + """Create backend state for a retained execution session.""" + raise NotImplementedError + + @abstractmethod + def resume_callback( + self, + session: BackendSession, + response: HostCallbackResponse, + ) -> "BackendExecutionContext": + """Resume a session execution with one host callback response.""" + raise NotImplementedError + + @abstractmethod + def close_session(self, session: BackendSession) -> None: + """Release resources owned by a retained execution session.""" + raise NotImplementedError + + +@dataclass(frozen=True) +class BackendHostCallbackRequest: + """Internal representation of a callback requested by executing code.""" + + request_id: str + request_type: str + name: str + arguments: dict[str, JsonValue] + @dataclass class BackendExecutionResult: @@ -63,22 +117,28 @@ class BackendExecutionResult: status: TaskStatus stdout: str = "" stderr: str = "" - structured_content: Any = None + structured_content: JsonValue | NotGiven = NOT_GIVEN + """Structured function result, or ``NOT_GIVEN`` when no result exists.""" error: str | None = None - metadata: dict[str, Any] | None = None + host_callback_request: BackendHostCallbackRequest | None = None + """Pending host callback request when status is ``input_required``.""" + metadata: dict[str, JsonValue] | None = None -class BackendExecutionContext: +class BackendExecutionContext(ABC): """Handle for one active or completed backend execution.""" + @abstractmethod def get_result(self) -> BackendExecutionResult: """Return the latest normalized result for this execution.""" raise NotImplementedError + @abstractmethod def wait(self) -> BackendExecutionResult: """Wait until the execution reaches a terminal or waiting state.""" raise NotImplementedError + @abstractmethod def cancel(self) -> BackendExecutionResult: """Request cancellation and return the resulting backend state.""" raise NotImplementedError @@ -95,6 +155,6 @@ def validate_function(self, source_code: str, function_name: str) -> None: """Validate source code and entry point for function execution.""" raise NotImplementedError - def build_namespace(self) -> dict[str, Any]: + def build_namespace(self) -> dict[str, object]: """Build the initial namespace for a worker execution.""" raise NotImplementedError diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/__init__.py b/wayflowcore/src/wayflowcore/codeserver/backends/__init__.py index 19149dbb8..8e67f3559 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backends/__init__.py +++ b/wayflowcore/src/wayflowcore/codeserver/backends/__init__.py @@ -1,4 +1,4 @@ -# Copyright © 2025 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py index 0f8d4c59e..bb1676484 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py +++ b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py @@ -1,43 +1,302 @@ -# Copyright © 2025 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. -"""Local Python execution backend and worker-process handles.""" +"""Local Python execution backend and queue-worker process handles.""" from __future__ import annotations +import multiprocessing +import os +import queue +import signal +import time +import uuid +from collections.abc import Mapping from dataclasses import dataclass, field -from typing import Any, Mapping +from multiprocessing.process import BaseProcess +from multiprocessing.queues import Queue +from threading import Lock +from typing import cast from wayflowcore.codeserver.backend import ( BackendExecutionContext, BackendExecutionResult, + BackendHostCallbackRequest, CodeExecutorBackend, PythonExecutionPolicy, ) +from wayflowcore.codeserver.backends.pythonworker import worker_main +from wayflowcore.codeserver.models import ( + TASK_STATUS_CANCELLED, + TASK_STATUS_COMPLETED, + TASK_STATUS_FAILED, + TASK_STATUS_INPUT_REQUIRED, + TASK_STATUS_TIMED_OUT, + HostCallbackResponse, + HostInteractions, + JsonValue, +) +from wayflowcore.codeserver.sessions import BackendSession + +WorkerCommand = dict[str, object] +WorkerMessage = dict[str, object] +@dataclass class LocalPythonBackend(CodeExecutorBackend): """Backend configuration for executing Python locally.""" policy: PythonExecutionPolicy | None = None + cancel_grace_seconds: float = 0.5 + """Time allowed for a worker to exit after a termination request.""" + + max_stdout_chars: int = 50_000 + """Maximum captured standard-output characters per execution.""" + + max_stderr_chars: int = 50_000 + """Maximum captured standard-error characters per execution.""" + + _sessions: dict[str, LocalPythonSession] = field(default_factory=dict, init=False, repr=False) + + def get_capabilities(self) -> dict[str, JsonValue]: + """Return capabilities supported by the local Python backend.""" + return { + "supported_languages": ["python"], + "execution_modes": ["script", "function"], + "supports_sessions": True, + "supports_host_interactions": True, + } - def start_script(self, source_code: str, *, session: Any = None) -> BackendExecutionContext: + def validate_language(self, language_id: str) -> None: + """Validate that the local backend supports Python.""" + if language_id != "python": + raise ValueError(f"Unsupported language: {language_id}") + + def start_script( + self, + source_code: str, + *, + session: BackendSession | None = None, + ) -> LocalPythonExecution: """Start a Python script in a worker process.""" - raise NotImplementedError + if self.policy is not None: + self.policy.validate_script(source_code) + return self._start_execution( + { + "type": "run", + "mode": "script", + "source_code": source_code, + }, + session=session, + ) def start_function( self, source_code: str, function_name: str, - arguments: Mapping[str, Any], + arguments: Mapping[str, JsonValue], *, - session: Any = None, - ) -> BackendExecutionContext: + session: BackendSession | None = None, + ) -> LocalPythonExecution: """Start a Python function in a worker process.""" - raise NotImplementedError + if self.policy is not None: + self.policy.validate_function(source_code, function_name) + return self._start_execution( + { + "type": "run", + "mode": "function", + "source_code": source_code, + "function_name": function_name, + "arguments": dict(arguments), + }, + session=session, + ) + + def create_session( + self, + session_id: str, + language_id: str, + *, + host_interactions: HostInteractions | None = None, + ) -> LocalPythonSession: + """Create one retained local Python worker session.""" + self.validate_language(language_id) + command_queue, result_queue, process = self._spawn_worker(session_mode=True) + session = LocalPythonSession( + session_id=session_id, + language_id=language_id, + process=process, + command_queue=command_queue, + result_queue=result_queue, + host_interactions=host_interactions, + ) + self._sessions[session_id] = session + return session + + def resume_callback( + self, + session: BackendSession, + response: HostCallbackResponse, + ) -> LocalPythonExecution: + """Resume a paused local Python session from a host response.""" + local_session = self._require_local_python_session(session) + with local_session.lock: + if local_session.pending_callback_request_id != response.request_id: + return self._failed_execution("Unknown host callback request id.") + if not local_session.is_active: + return self._failed_execution("Session is no longer active.") + local_session.pending_callback_request_id = None + execution_id = self._create_new_execution_id() + local_session.active_execution_id = execution_id + local_session.command_queue.put( + { + "type": "callback_response", + "request_id": response.request_id, + "result": response.result, + } + ) + return LocalPythonExecution( + execution_id=execution_id, + backend=self, + process=local_session.process, + command_queue=local_session.command_queue, + result_queue=local_session.result_queue, + session=local_session, + deadline=time.monotonic() + self.execution_timeout_seconds, + ) + + def close_session(self, session: BackendSession) -> None: + """Close a retained local Python worker session.""" + local_session = self._require_local_python_session(session) + with local_session.lock: + if local_session.closed: + return + local_session.close() + local_session.process.join(self.cancel_grace_seconds) + if local_session.process.is_alive(): + self._terminate_process(local_session.process) + _close_queue(local_session.command_queue) + _close_queue(local_session.result_queue) + self._sessions.pop(session.session_id, None) + + def close_all_sessions(self) -> None: + """Close every session still owned by this backend instance.""" + for session in list(self._sessions.values()): + self.close_session(session) + + def _start_execution( + self, + command: WorkerCommand, + *, + session: BackendSession | None, + ) -> LocalPythonExecution: + """Start one worker command statelessly or in a retained session.""" + command.update( + { + "max_stdout_chars": self.max_stdout_chars, + "max_stderr_chars": self.max_stderr_chars, + } + ) + if session is None: + command["host_interactions_enabled"] = False # only sessions can use host interactions + command_queue, result_queue, process = self._spawn_worker(session_mode=False) + command_queue.put(command) + return LocalPythonExecution( + execution_id=self._create_new_execution_id(), + backend=self, + process=process, + command_queue=command_queue, + result_queue=result_queue, + deadline=time.monotonic() + self.execution_timeout_seconds, + ) + + if session.language_id != "python": + return self._failed_execution("Session language does not match the Python backend.") + local_session = self._require_local_python_session(session) + with local_session.lock: + if not local_session.is_active: + return self._failed_execution("Session is no longer active.") + if local_session.active_execution_id is not None: + return self._failed_execution("Session already has an active execution.") + if not local_session.process.is_alive(): + local_session.failed = True + return self._failed_execution("Session worker exited unexpectedly.") + execution_id = self._create_new_execution_id() + local_session.active_execution_id = execution_id + command["host_interactions_enabled"] = _host_interactions_enabled( + local_session.host_interactions + ) + local_session.command_queue.put(command) + return LocalPythonExecution( + execution_id=execution_id, + backend=self, + process=local_session.process, + command_queue=local_session.command_queue, + result_queue=local_session.result_queue, + session=local_session, + deadline=time.monotonic() + self.execution_timeout_seconds, + ) + + def _spawn_worker( + self, *, session_mode: bool + ) -> tuple[Queue[object], Queue[object], BaseProcess]: + """Create and start one local Python worker process.""" + context = multiprocessing.get_context("spawn") + command_queue = cast(Queue[object], context.Queue()) + result_queue = cast(Queue[object], context.Queue()) + process = context.Process( + target=worker_main, + args=(command_queue, result_queue, self.policy), + kwargs={"session_mode": session_mode}, + ) + process.daemon = True + process.start() + return command_queue, result_queue, process + + def _terminate_process(self, process: BaseProcess) -> None: + """Terminate a worker process and its descendants when supported.""" + if not process.is_alive(): + return + if os.name == "posix" and process.pid is not None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + process.terminate() + else: + process.terminate() + process.join(self.cancel_grace_seconds) + if not process.is_alive(): + return + if os.name == "posix" and process.pid is not None: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + process.kill() + else: + process.kill() + process.join() + + def _require_local_python_session(self, session: BackendSession) -> LocalPythonSession: + """Require a session owned by this local Python backend.""" + if not isinstance(session, LocalPythonSession): + raise ValueError("Session does not belong to the local Python backend.") + return session + + def _failed_execution(self, error: str) -> LocalPythonExecution: + """Create an execution handle already resolved to failure.""" + return LocalPythonExecution( + execution_id=self._create_new_execution_id(), + backend=self, + result=BackendExecutionResult(status=TASK_STATUS_FAILED, error=error), + ) + + @staticmethod + def _create_new_execution_id() -> str: + """Create one backend-local execution identifier.""" + return f"exec_{uuid.uuid4().hex}" @dataclass @@ -45,29 +304,277 @@ class LocalPythonExecution(BackendExecutionContext): """Handle for one local Python worker execution.""" execution_id: str + backend: LocalPythonBackend = field(repr=False) + process: BaseProcess | None = field(default=None, repr=False) + command_queue: Queue[object] | None = field(default=None, repr=False) + result_queue: Queue[object] | None = field(default=None, repr=False) + session: LocalPythonSession | None = field(default=None, repr=False) + deadline: float | None = field(default=None, repr=False) result: BackendExecutionResult | None = None def get_result(self) -> BackendExecutionResult: - """Return the latest worker result.""" - raise NotImplementedError + """Return the latest worker result without blocking.""" + if self.result is None: + self._consume_available_messages() + self._check_timeout() + self._check_worker_exit() + return self.result or BackendExecutionResult(status="working") def wait(self) -> BackendExecutionResult: """Wait for the worker to finish or request host input.""" - raise NotImplementedError + while self.result is None: + self._check_timeout() + if self.result is not None: + break + self._wait_for_message(self._remaining_timeout()) + self._check_worker_exit() + if self.result is None: + raise RuntimeError("Execution worker exited without producing a result.") + return self.result def cancel(self) -> BackendExecutionResult: """Terminate the worker and return its cancellation result.""" - raise NotImplementedError + if self.result is not None: + return self.result + if self.process is not None: + self.backend._terminate_process(self.process) + if self.session is not None: + self.session.failed = True + self._set_terminal_result(BackendExecutionResult(status=TASK_STATUS_CANCELLED)) + if self.result is None: + raise RuntimeError("Execution cancellation did not produce a result.") + return self.result + + def is_alive(self) -> bool: + """Return whether the backing worker process is still alive.""" + return self.process is not None and self.process.is_alive() + + def _consume_available_messages(self) -> None: + """Update the current result from messages already sent by the worker.""" + if self.result_queue is None: + return + while self.result is None: + try: + message = self.result_queue.get_nowait() + except queue.Empty: + return + self._handle_message(message) + + def _wait_for_message(self, timeout_seconds: float | None) -> None: + """Wait for one worker message and update the current result.""" + if self.result_queue is None: + self._set_terminal_result( + BackendExecutionResult( + status=TASK_STATUS_FAILED, error="Execution has no worker queue." + ) + ) + return + try: + message = self.result_queue.get(timeout=timeout_seconds) + except queue.Empty: + self._check_timeout() + return + self._handle_message(message) + + def _handle_message(self, raw_message: object) -> None: + """Convert one worker message into a normalized backend result.""" + if not isinstance(raw_message, dict) or not all( + isinstance(key, str) for key in raw_message + ): + self._set_terminal_result( + BackendExecutionResult( + status=TASK_STATUS_FAILED, error="Worker returned an invalid message." + ) + ) + return + message = cast(WorkerMessage, raw_message) + message_type = message.get("type") + stdout = _optional_str(message.get("stdout")) + stderr = _optional_str(message.get("stderr")) + metadata = _truncation_metadata(message) + if message_type == "completed": + if "structured_content" in message: + structured_content = _json_value(message["structured_content"]) + self._set_terminal_result( + BackendExecutionResult( + status=TASK_STATUS_COMPLETED, + stdout=stdout, + stderr=stderr, + structured_content=structured_content, + metadata=metadata, + ) + ) + else: + self._set_terminal_result( + BackendExecutionResult( + status=TASK_STATUS_COMPLETED, + stdout=stdout, + stderr=stderr, + metadata=metadata, + ) + ) + return + if message_type == "failed": + self._set_terminal_result( + BackendExecutionResult( + status=TASK_STATUS_FAILED, + stdout=stdout, + stderr=stderr, + error=_optional_str(message.get("error")) or "Python execution failed.", + metadata=metadata, + ) + ) + return + if message_type == "host_request": + request = BackendHostCallbackRequest( + request_id=_required_str(message, "request_id"), + request_type=_required_str(message, "request_type"), + name=_required_str(message, "name"), + arguments=_json_object(message.get("arguments")), + ) + if self.session is None: + self._set_terminal_result( + BackendExecutionResult( + status=TASK_STATUS_FAILED, + error="Host callbacks require a retained session.", + ) + ) + return + self.session.pending_callback_request_id = request.request_id + self.result = BackendExecutionResult( + status=TASK_STATUS_INPUT_REQUIRED, + stdout=stdout, + stderr=stderr, + host_callback_request=request, + metadata=metadata, + ) + return + self._set_terminal_result( + BackendExecutionResult( + status=TASK_STATUS_FAILED, error="Worker returned an unknown message." + ) + ) + + def _check_timeout(self) -> None: + """Terminate an execution whose parent-side deadline has elapsed.""" + if self.result is not None or self.deadline is None or time.monotonic() < self.deadline: + return + if self.process is not None: + self.backend._terminate_process(self.process) + if self.session is not None: + self.session.failed = True + self._set_terminal_result(BackendExecutionResult(status=TASK_STATUS_TIMED_OUT)) + + def _check_worker_exit(self) -> None: + """Report unexpected worker exit when no result has been received.""" + if self.result is None and self.process is not None and not self.process.is_alive(): + self._set_terminal_result( + BackendExecutionResult( + status=TASK_STATUS_FAILED, error="Python worker exited unexpectedly." + ) + ) + + def _remaining_timeout(self) -> float | None: + """Return time remaining until this execution reaches its deadline.""" + if self.deadline is None: + return None + return max(0.0, self.deadline - time.monotonic()) + + def _set_terminal_result(self, result: BackendExecutionResult) -> None: + """Store a terminal result and release a session for later commands.""" + self.result = result + if self.session is not None and result.status != TASK_STATUS_INPUT_REQUIRED: + with self.session.lock: + if self.session.active_execution_id == self.execution_id: + self.session.active_execution_id = None + if self.session is None and result.status != TASK_STATUS_INPUT_REQUIRED: + if self.process is not None: + self.process.join(self.backend.cancel_grace_seconds) + if self.command_queue is not None: + _close_queue(self.command_queue) + if self.result_queue is not None: + _close_queue(self.result_queue) @dataclass -class LocalPythonSession: +class LocalPythonSession(BackendSession): """Handle for a long-lived local Python session worker.""" session_id: str - process: Any = None - lock: Any = field(default=None, repr=False) + language_id: str + process: BaseProcess + command_queue: Queue[object] + result_queue: Queue[object] + host_interactions: HostInteractions | None = None + lock: Lock = field(default_factory=Lock, repr=False) + pending_callback_request_id: str | None = None + active_execution_id: str | None = None + closed: bool = False + failed: bool = False + + @property + def is_active(self) -> bool: + """Return whether this session can accept another worker command.""" + return not self.closed and not self.failed def close(self) -> None: - """Terminate the session worker and release its resources.""" - raise NotImplementedError + """Request graceful closure of the retained worker.""" + if not self.closed and self.process.is_alive(): + self.command_queue.put({"type": "close"}) + self.closed = True + + +def _host_interactions_enabled(host_interactions: HostInteractions | None) -> bool: + """Return whether tool-execution callbacks are allowed for a session.""" + return ( + host_interactions is not None + and host_interactions.enabled + and "tool_execution" in host_interactions.allowed_request_types + ) + + +def _optional_str(value: object) -> str: + """Return a string value or an empty string for an absent invalid field.""" + return value if isinstance(value, str) else "" + + +def _required_str(message: Mapping[str, object], name: str) -> str: + """Return one required string worker-message field.""" + value = message.get(name) + if not isinstance(value, str): + raise ValueError(f"Worker message field '{name}' must be a string.") + return value + + +def _json_object(value: object) -> dict[str, JsonValue]: + """Return one JSON object value or raise for invalid worker data.""" + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ValueError("Worker callback arguments must be a JSON object.") + return {key: _json_value(item) for key, item in value.items()} + + +def _json_value(value: object) -> JsonValue: + """Validate a value recursively as JSON-compatible worker data.""" + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, list): + return [_json_value(item) for item in value] + if isinstance(value, dict) and all(isinstance(key, str) for key in value): + return {key: _json_value(item) for key, item in value.items()} + raise ValueError("Worker returned a non-JSON-compatible value.") + + +def _truncation_metadata(message: Mapping[str, object]) -> dict[str, JsonValue] | None: + """Return output-truncation metadata when the worker reported it.""" + metadata: dict[str, JsonValue] = {} + if message.get("stdout_truncated") is True: + metadata["stdout_truncated"] = True + if message.get("stderr_truncated") is True: + metadata["stderr_truncated"] = True + return metadata or None + + +def _close_queue(worker_queue: Queue[object]) -> None: + """Close one parent-owned multiprocessing queue and its feeder thread.""" + worker_queue.close() + worker_queue.join_thread() diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py b/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py index c860694b4..6cfea4b3b 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py +++ b/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py @@ -1,34 +1,267 @@ -# Copyright © 2025 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. -"""Worker process entry point for the local Python backend.""" +"""Queue-based worker process for the local Python execution backend.""" from __future__ import annotations -from typing import Any +import contextlib +import io +import json +import os +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from multiprocessing.queues import Queue +from typing import cast +from wayflowcore._utils.notgiven import NOT_GIVEN, NotGiven +from wayflowcore.codeserver.backend import PythonExecutionPolicy +from wayflowcore.codeserver.models import JsonValue -def run_script(source_code: str, namespace: dict[str, Any]) -> Any: +WorkerCommand = dict[str, object] +WorkerMessage = dict[str, object] + + +class _OutputCapture(io.TextIOBase): + """Bounded text capture for one execution stream.""" + + def __init__(self, limit: int) -> None: + self._limit = limit + self._parts: list[str] = [] + self._size = 0 + self.truncated = False + + def writable(self) -> bool: + """Report that text may be written to this capture.""" + return True + + def write(self, text: str) -> int: + """Capture text up to the configured limit.""" + available = self._limit - self._size + if available > 0: + captured = text[:available] + self._parts.append(captured) + self._size += len(captured) + if len(text) > available: + self.truncated = True + return len(text) + + @property + def text(self) -> str: + """Return the captured text.""" + return "".join(self._parts) + + +@dataclass +class _HostBridge: + """Object injected into a session namespace for host callbacks.""" + + command_queue: Queue[object] + result_queue: Queue[object] + stdout: _OutputCapture + stderr: _OutputCapture + + def tool_execution(self, name: str, /, **arguments: JsonValue) -> JsonValue: + """Request one host tool execution and wait for its matching response.""" + request_id = f"callback_{uuid.uuid4().hex}" + self.result_queue.put( + { + "type": "host_request", + "request_id": request_id, + "request_type": "tool_execution", + "name": name, + "arguments": arguments, + "stdout": self.stdout.text, + "stderr": self.stderr.text, + } + ) + + while True: + command = _require_command(self.command_queue.get()) + if command.get("type") == "close": + raise RuntimeError("Session closed while waiting for a host callback response.") + if command.get("type") != "callback_response": + raise RuntimeError("Expected a host callback response.") + if command.get("request_id") != request_id: + raise RuntimeError("Host callback response did not match the pending request.") + return _json_value(command.get("result")) + + +def run_script(source_code: str, namespace: dict[str, object]) -> None: """Execute script source code in a worker namespace.""" - raise NotImplementedError + compiled = compile(source_code, filename="", mode="exec") + exec(compiled, namespace, namespace) # nosec B102 - intentional code executor behavior. def run_function( source_code: str, function_name: str, - arguments: dict[str, Any], - namespace: dict[str, Any], -) -> Any: + arguments: Mapping[str, JsonValue], + namespace: dict[str, object], +) -> object: """Define source code and invoke one named function in a worker namespace.""" - raise NotImplementedError + run_script(source_code, namespace) + function = namespace.get(function_name) + if not callable(function): + raise ValueError(f"Function '{function_name}' is not defined.") + return function(**arguments) + + +def execute_command( + command: WorkerCommand, + *, + namespace: dict[str, object], + command_queue: Queue[object], + result_queue: Queue[object], +) -> bool: + """Execute one worker command and return whether the loop should continue.""" + command_type = command.get("type") + if command_type == "close": + return False + if command_type != "run": + _emit_failed(result_queue, "Unsupported worker command.") + return True + + source_code = _required_str(command, "source_code") + mode = _required_str(command, "mode") + max_stdout_chars = _required_non_negative_int(command, "max_stdout_chars") + max_stderr_chars = _required_non_negative_int(command, "max_stderr_chars") + stdout = _OutputCapture(max_stdout_chars) + stderr = _OutputCapture(max_stderr_chars) + + if command.get("host_interactions_enabled") is True: + namespace["host"] = _HostBridge(command_queue, result_queue, stdout, stderr) + else: + namespace.pop("host", None) + + try: + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): # type: ignore[type-var] + if mode == "script": + run_script(source_code, namespace) + result_queue.put(_completed_message(stdout, stderr)) + elif mode == "function": + function_name = _required_str(command, "function_name") + arguments = _json_object(command.get("arguments"), "arguments") + result = run_function(source_code, function_name, arguments, namespace) + result_queue.put( + _completed_message(stdout, stderr, structured_content=_json_value(result)) + ) + else: + raise ValueError("Execution mode must be 'script' or 'function'.") + except Exception as exc: # noqa: BLE001 - user-code boundary. + _emit_failed(result_queue, str(exc) or type(exc).__name__, stdout, stderr) + return True + + +def worker_main( + command_queue: Queue[object], + result_queue: Queue[object], + policy: PythonExecutionPolicy | None, + *, + session_mode: bool, +) -> None: + """Run the queue-based local Python worker command loop.""" + if os.name == "posix": + os.setsid() + + namespace = {} if policy is None else policy.build_namespace() + while True: + command = _require_command(command_queue.get()) + if not execute_command( + command, + namespace=namespace, + command_queue=command_queue, + result_queue=result_queue, + ): + return + if not session_mode: + return + + +def _completed_message( + stdout: _OutputCapture, + stderr: _OutputCapture, + *, + structured_content: JsonValue | NotGiven = NOT_GIVEN, +) -> WorkerMessage: + """Create one completed worker message.""" + message: WorkerMessage = { + "type": "completed", + "stdout": stdout.text, + "stderr": stderr.text, + "stdout_truncated": stdout.truncated, + "stderr_truncated": stderr.truncated, + } + if structured_content is not NOT_GIVEN: + message["structured_content"] = structured_content + return message + + +def _emit_failed( + result_queue: Queue[object], + error: str, + stdout: _OutputCapture | None = None, + stderr: _OutputCapture | None = None, +) -> None: + """Send one normalized worker failure message.""" + result_queue.put( + { + "type": "failed", + "error": error, + "stdout": "" if stdout is None else stdout.text, + "stderr": "" if stderr is None else stderr.text, + "stdout_truncated": False if stdout is None else stdout.truncated, + "stderr_truncated": False if stderr is None else stderr.truncated, + } + ) + + +def _require_command(value: object) -> WorkerCommand: + """Validate a queue value as a worker command.""" + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise RuntimeError("Worker command must be an object with string keys.") + return cast(WorkerCommand, value) + + +def _required_str(command: Mapping[str, object], name: str) -> str: + """Return one required string field from a worker command.""" + value = command.get(name) + if not isinstance(value, str): + raise ValueError(f"Worker command field '{name}' must be a string.") + return value + + +def _required_non_negative_int(command: Mapping[str, object], name: str) -> int: + """Return one required non-negative integer field from a worker command.""" + value = command.get(name) + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f"Worker command field '{name}' must be a non-negative integer.") + return value + + +def _json_object(value: object, name: str) -> dict[str, JsonValue]: + """Validate an object as a JSON object.""" + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ValueError(f"Worker command field '{name}' must be an object.") + return {key: _json_value(item) for key, item in value.items()} + + +def _json_value(value: object) -> JsonValue: + """Return a JSON-compatible copy of one value or raise ``ValueError``.""" + try: + encoded = json.dumps(value, allow_nan=False, separators=(",", ":")) + except (TypeError, ValueError) as exc: + raise ValueError("Value is not JSON-compatible.") from exc + return cast(JsonValue, json.loads(encoded)) def main() -> None: - """Run the local Python worker command loop.""" - raise NotImplementedError + """Reject direct module execution without worker queue arguments.""" + raise SystemExit("This module is launched by LocalPythonBackend through multiprocessing.") if __name__ == "__main__": diff --git a/wayflowcore/src/wayflowcore/codeserver/models.py b/wayflowcore/src/wayflowcore/codeserver/models.py index e0e36c434..93c1619a7 100644 --- a/wayflowcore/src/wayflowcore/codeserver/models.py +++ b/wayflowcore/src/wayflowcore/codeserver/models.py @@ -1,4 +1,4 @@ -# Copyright © 2025 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License @@ -9,9 +9,16 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Final, Literal, TypeAlias +from typing import Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import TypeAliasType + +JsonValue = TypeAliasType( + "JsonValue", + "None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue]", +) +"""A JSON-compatible value.""" TaskStatus: TypeAlias = Literal[ "working", "input_required", "completed", "failed", "cancelled", "timed_out" @@ -44,7 +51,7 @@ class FunctionInput(CodeExecutorModel): type: Literal["function"] source_code: str function_name: str - arguments: dict[str, Any] + arguments: dict[str, JsonValue] """Named JSON-compatible arguments passed to the function.""" @@ -53,7 +60,7 @@ class HostCallbackResponse(CodeExecutorModel): type: Literal["host_response"] request_id: str - result: Any = None + result: JsonValue = None """JSON-compatible result returned by the host.""" @@ -80,7 +87,7 @@ class CodeExecutionRequest(CodeExecutorModel): """Dependencies expected to be available in the execution environment.""" session_id: str | None = None - metadata: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, JsonValue] = Field(default_factory=dict) wait: bool = True @model_validator(mode="after") @@ -97,7 +104,7 @@ class ExecutionResult(CodeExecutorModel): content: list[TextContent] = Field(default_factory=list) """Content blocks such as captured text output.""" - structured_content: Any = Field(default=None, alias="structuredContent") + structured_content: JsonValue = Field(default=None, alias="structuredContent") """Optional JSON-compatible structured result, including scalar values and null.""" is_error: bool = Field(default=False, alias="isError") @@ -111,7 +118,7 @@ class HostCallbackRequest(CodeExecutorModel): request_id: str request_type: Literal["tool_execution"] name: str - arguments: dict[str, Any] + arguments: dict[str, JsonValue] ExecutionOutputItem: TypeAlias = ExecutionResult | HostCallbackRequest @@ -128,7 +135,7 @@ class ExecutionResponse(CodeExecutorModel): completed_at: datetime | None = None language_id: str output: list[ExecutionOutputItem] = Field(default_factory=list) - metadata: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, JsonValue] = Field(default_factory=dict) class HostInteractions(CodeExecutorModel): @@ -143,7 +150,7 @@ class CreateSessionRequest(CodeExecutorModel): language_id: str host_interactions: HostInteractions | None = None - metadata: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, JsonValue] = Field(default_factory=dict) class SessionSnapshot(CodeExecutorModel): @@ -154,4 +161,13 @@ class SessionSnapshot(CodeExecutorModel): status: Literal["active", "closing", "closed", "expired"] language_id: str host_interactions: HostInteractions | None = None - metadata: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, JsonValue] = Field(default_factory=dict) + + +class CodeExecutorCapabilities(CodeExecutorModel): + """Public capabilities advertised by a Code Executor server.""" + + view: Literal["public"] = "public" + protocol_version: str + server_name: str + capabilities: dict[str, JsonValue] = Field(default_factory=dict) diff --git a/wayflowcore/src/wayflowcore/codeserver/server.py b/wayflowcore/src/wayflowcore/codeserver/server.py new file mode 100644 index 000000000..718eafe2f --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/server.py @@ -0,0 +1,49 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Server composition and process entry point for Code Executor Protocol.""" + +from __future__ import annotations + +from fastapi import FastAPI + +from wayflowcore.codeserver.app import create_code_executor_app +from wayflowcore.codeserver.backend import CodeExecutorBackend +from wayflowcore.codeserver.backends.local_python import LocalPythonBackend +from wayflowcore.codeserver.service import CodeExecutionService +from wayflowcore.codeserver.storage import CodeExecutorStorage + + +class CodeExecutorServer: + """Compose a Code Executor service and expose it as a FastAPI application.""" + + def __init__( + self, + backend: CodeExecutorBackend | None = None, + storage: CodeExecutorStorage | None = None, + *, + server_name: str = "wayflow-code-server", + protocol_version: str = "26.1.3", + ) -> None: + """Initialize a server with a backend and optional snapshot storage.""" + self.backend = backend or LocalPythonBackend() + self.service = CodeExecutionService(backend=self.backend, storage=storage) + self.server_name = server_name + self.protocol_version = protocol_version + + def get_app(self) -> FastAPI: + """Return the FastAPI application for deployment by an ASGI server.""" + return create_code_executor_app( + self.service, + server_name=self.server_name, + protocol_version=self.protocol_version, + ) + + def run(self, host: str = "127.0.0.1", port: int = 8765) -> None: + """Run the Code Executor server with Uvicorn.""" + import uvicorn + + uvicorn.run(self.get_app(), host=host, port=port, reload=False) diff --git a/wayflowcore/src/wayflowcore/codeserver/service.py b/wayflowcore/src/wayflowcore/codeserver/service.py index 726425068..20ed4ad78 100644 --- a/wayflowcore/src/wayflowcore/codeserver/service.py +++ b/wayflowcore/src/wayflowcore/codeserver/service.py @@ -1,4 +1,4 @@ -# Copyright © 2025 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License @@ -8,19 +8,45 @@ from __future__ import annotations -from wayflowcore.codeserver.backend import CodeExecutorBackend +from datetime import datetime, timezone +from uuid import uuid4 + +from wayflowcore._utils.notgiven import NOT_GIVEN +from wayflowcore.codeserver.backend import ( + BackendExecutionContext, + BackendExecutionResult, + CodeExecutorBackend, +) from wayflowcore.codeserver.models import ( + TASK_STATUS_CANCELLED, + TASK_STATUS_COMPLETED, + TASK_STATUS_FAILED, + TASK_STATUS_INPUT_REQUIRED, + TASK_STATUS_TIMED_OUT, + TASK_STATUS_WORKING, CodeExecutionRequest, CreateSessionRequest, ExecutionResponse, + ExecutionResult, + FunctionInput, + HostCallbackRequest, + HostCallbackResponse, + JsonValue, + ScriptInput, SessionSnapshot, + TextContent, ) +from wayflowcore.codeserver.storage import CodeExecutorStorage class CodeExecutionService: """Coordinates execution requests, storage, sessions, and a backend.""" - def __init__(self, backend: CodeExecutorBackend) -> None: + def __init__( + self, + backend: CodeExecutorBackend, + storage: CodeExecutorStorage | None = None, + ) -> None: """Initialize the service with an execution backend. Parameters @@ -29,6 +55,8 @@ def __init__(self, backend: CodeExecutorBackend) -> None: Backend responsible for running code. """ self.backend = backend + self.storage = storage or CodeExecutorStorage() + self._execution_contexts: dict[str, BackendExecutionContext] = {} def execute(self, request: CodeExecutionRequest) -> ExecutionResponse: """Create an execution and optionally wait for its completion. @@ -43,7 +71,13 @@ def execute(self, request: CodeExecutionRequest) -> ExecutionResponse: ExecutionResponse Current execution snapshot. """ - raise NotImplementedError + if not request.wait: + return self.create_execution(request) + + initial_response = self.create_execution(request) + context = self._execution_contexts[initial_response.id] + result = context.wait() + return self._update_execution(initial_response, result) def create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: """Create an execution without waiting for completion. @@ -58,7 +92,128 @@ def create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: ExecutionResponse Initial execution snapshot. """ - raise NotImplementedError + context = self._start_execution(request) + response = ExecutionResponse( + id=self._create_execution_id(), + object="response", + created_at=datetime.now(timezone.utc), + status=TASK_STATUS_WORKING, + language_id=request.language_id, + metadata=request.metadata, + ) + self.storage.create_execution(response) + self._execution_contexts[response.id] = context + return response + + def _start_execution(self, request: CodeExecutionRequest) -> BackendExecutionContext: + """Start one stateless script or function request on the backend.""" + self.backend.validate_language(request.language_id) + if request.session_id is not None: + raise NotImplementedError("Session execution is not implemented yet.") + + input_item = request.input[0] + if isinstance(input_item, ScriptInput): + return self.backend.start_script(input_item.source_code) + if isinstance(input_item, FunctionInput): + return self.backend.start_function( + input_item.source_code, + input_item.function_name, + input_item.arguments, + ) + if isinstance(input_item, HostCallbackResponse): + raise ValueError("Host callback responses require a session.") + raise TypeError(f"Unsupported execution input: {type(input_item).__name__}") + + def _update_execution( + self, + previous_response: ExecutionResponse, + result: BackendExecutionResult, + ) -> ExecutionResponse: + """Update one stored response from its latest backend result.""" + response = self._result_to_response( + execution_id=previous_response.id, + created_at=previous_response.created_at, + language_id=previous_response.language_id, + result=result, + request_metadata=previous_response.metadata, + completed_at=previous_response.completed_at, + ) + self.storage.update_execution(response) + return response + + @staticmethod + def _create_execution_id() -> str: + """Create one public execution identifier.""" + return f"exec_{uuid4().hex}" + + def _result_to_response( + self, + *, + execution_id: str, + created_at: datetime, + language_id: str, + result: BackendExecutionResult, + request_metadata: dict[str, JsonValue], + completed_at: datetime | None = None, + ) -> ExecutionResponse: + """Convert one backend result into a protocol execution snapshot.""" + output = [] if result.status == TASK_STATUS_WORKING else [self._result_to_output(result)] + if completed_at is None and result.status in { + TASK_STATUS_COMPLETED, + TASK_STATUS_FAILED, + TASK_STATUS_TIMED_OUT, + TASK_STATUS_CANCELLED, + }: + completed_at = datetime.now(timezone.utc) + metadata = dict(request_metadata) + if result.metadata: + metadata.update(result.metadata) + if result.error is not None: + metadata["error"] = result.error + return ExecutionResponse( + id=execution_id, + object="response", + created_at=created_at, + status=result.status, + completed_at=completed_at, + language_id=language_id, + output=output, + metadata=metadata, + ) + + @staticmethod + def _result_to_output( + result: BackendExecutionResult, + ) -> ExecutionResult | HostCallbackRequest: + """Convert backend output and callback data into a protocol output item.""" + if result.status == TASK_STATUS_INPUT_REQUIRED: + if result.host_callback_request is None: + raise ValueError("Backend requested input without a callback request.") + callback = result.host_callback_request + return HostCallbackRequest( + type="host_request", + request_id=callback.request_id, + request_type=callback.request_type, + name=callback.name, + arguments=callback.arguments, + ) + + content: list[TextContent] = [] + if result.stdout: + content.append(TextContent(type="text", stream="stdout", text=result.stdout)) + if result.stderr: + content.append(TextContent(type="text", stream="stderr", text=result.stderr)) + structured_fields = ( + {"structured_content": result.structured_content} + if result.structured_content is not NOT_GIVEN + else {} + ) + return ExecutionResult( + type="output", + content=content, + is_error=result.status != TASK_STATUS_COMPLETED, + **structured_fields, + ) def get_execution(self, execution_id: str) -> ExecutionResponse: """Return the latest snapshot for an execution. @@ -73,7 +228,14 @@ def get_execution(self, execution_id: str) -> ExecutionResponse: ExecutionResponse Latest execution snapshot. """ - raise NotImplementedError + response = self.storage.get_execution(execution_id) + context = self._execution_contexts.get(execution_id) + if context is None or response.status not in { + TASK_STATUS_WORKING, + TASK_STATUS_INPUT_REQUIRED, + }: + return response + return self._update_execution(response, context.get_result()) def cancel_execution(self, execution_id: str) -> ExecutionResponse: """Request cancellation of an execution. @@ -88,7 +250,11 @@ def cancel_execution(self, execution_id: str) -> ExecutionResponse: ExecutionResponse Execution snapshot after cancellation is requested. """ - raise NotImplementedError + response = self.storage.get_execution(execution_id) + context = self._execution_contexts.get(execution_id) + if context is None or response.status not in {"working", TASK_STATUS_INPUT_REQUIRED}: + return response + return self._update_execution(response, context.cancel()) def create_session(self, request: CreateSessionRequest) -> SessionSnapshot: """Create a stateful execution session. diff --git a/wayflowcore/src/wayflowcore/codeserver/sessions.py b/wayflowcore/src/wayflowcore/codeserver/sessions.py index a1084e6c0..b5b2550b1 100644 --- a/wayflowcore/src/wayflowcore/codeserver/sessions.py +++ b/wayflowcore/src/wayflowcore/codeserver/sessions.py @@ -1,39 +1,51 @@ -# Copyright © 2025 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. -"""Backend session abstractions for stateful execution.""" +"""Backend-owned session abstractions for stateful execution.""" from __future__ import annotations -from dataclasses import dataclass -from typing import Any +from abc import ABC, abstractmethod +from wayflowcore.codeserver.models import HostInteractions -@dataclass -class BackendSessionState: - """Runtime state owned by one stateful backend session.""" + +class BackendSession(ABC): + """Resource handle for one backend-owned execution session.""" session_id: str + """Identifier assigned to this session by the service.""" + language_id: str - runtime: Any = None - closed: bool = False - failed: bool = False + """Language accepted by this session.""" + + host_interactions: HostInteractions | None + """Host callback configuration selected when the session was created.""" + + @property + @abstractmethod + def is_active(self) -> bool: + """Return whether the session can accept another execution.""" + + @abstractmethod + def close(self) -> None: + """Release resources owned by this session.""" class SessionRegistry: - """Registry for backend-owned stateful sessions.""" + """Registry storing backend-owned execution sessions by identifier.""" - def create(self, session_id: str, language_id: str) -> BackendSessionState: - """Create and register a backend session.""" + def add(self, session: BackendSession) -> None: + """Register one backend-created session.""" raise NotImplementedError - def get(self, session_id: str) -> BackendSessionState: + def get(self, session_id: str) -> BackendSession: """Return a registered backend session.""" raise NotImplementedError - def close(self, session_id: str) -> None: - """Close and remove a backend session.""" + def remove(self, session_id: str) -> BackendSession: + """Remove and return one registered backend session.""" raise NotImplementedError diff --git a/wayflowcore/src/wayflowcore/codeserver/storage.py b/wayflowcore/src/wayflowcore/codeserver/storage.py new file mode 100644 index 000000000..e46b19a60 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/storage.py @@ -0,0 +1,138 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Datastore-backed persistence for Code Executor Protocol snapshots.""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field + +from wayflowcore.codeserver.models import ExecutionResponse, SessionSnapshot +from wayflowcore.datastore import Datastore, Entity, InMemoryDatastore +from wayflowcore.property import StringProperty + +_EXECUTIONS_COLLECTION = "code_executions" +_SESSIONS_COLLECTION = "code_sessions" +_ID_PROPERTY = "id" +_DATA_PROPERTY = "data" + + +def _storage_schema() -> dict[str, Entity]: + """Build the datastore schema used for execution and session snapshots.""" + return { + _EXECUTIONS_COLLECTION: Entity( + properties={ + _ID_PROPERTY: StringProperty(), + _DATA_PROPERTY: StringProperty(), + } + ), + _SESSIONS_COLLECTION: Entity( + properties={ + _ID_PROPERTY: StringProperty(), + _DATA_PROPERTY: StringProperty(), + } + ), + } + + +@dataclass +class CodeExecutorStorage: + """Persist protocol snapshots through a configured :class:`Datastore`. + + Parameters + ---------- + datastore: + Datastore used for persistence. When omitted, an + :class:`InMemoryDatastore` is created for local development and tests. + """ + + datastore: Datastore | None = None + _datastore: Datastore = field(init=False, repr=False) + + def __post_init__(self) -> None: + """Initialize the configured or default datastore.""" + if self.datastore is not None: + self._datastore = self.datastore + return + # The storage adapter explicitly documents this as its local default; + # avoid repeating InMemoryDatastore's general-purpose warning here. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="InMemoryDatastore is for DEVELOPMENT", + category=UserWarning, + ) + self._datastore = InMemoryDatastore(_storage_schema()) + + def create_execution(self, response: ExecutionResponse) -> ExecutionResponse: + """Persist and return a new execution snapshot.""" + self._create(_EXECUTIONS_COLLECTION, response.id, response.model_dump_json(by_alias=True)) + return response + + def get_execution(self, execution_id: str) -> ExecutionResponse: + """Retrieve an execution snapshot by identifier.""" + data = self._get_data(_EXECUTIONS_COLLECTION, execution_id) + return ExecutionResponse.model_validate_json(data) + + def update_execution(self, response: ExecutionResponse) -> ExecutionResponse: + """Replace an existing execution snapshot.""" + self._update(_EXECUTIONS_COLLECTION, response.id, response.model_dump_json(by_alias=True)) + return response + + def delete_execution(self, execution_id: str) -> None: + """Delete an execution snapshot.""" + self._datastore.delete(_EXECUTIONS_COLLECTION, where={_ID_PROPERTY: execution_id}) + + def create_session(self, snapshot: SessionSnapshot) -> SessionSnapshot: + """Persist and return a new session snapshot.""" + self._create(_SESSIONS_COLLECTION, snapshot.id, snapshot.model_dump_json(by_alias=True)) + return snapshot + + def get_session(self, session_id: str) -> SessionSnapshot: + """Retrieve a session snapshot by identifier.""" + data = self._get_data(_SESSIONS_COLLECTION, session_id) + return SessionSnapshot.model_validate_json(data) + + def update_session(self, snapshot: SessionSnapshot) -> SessionSnapshot: + """Replace an existing session snapshot.""" + self._update(_SESSIONS_COLLECTION, snapshot.id, snapshot.model_dump_json(by_alias=True)) + return snapshot + + def delete_session(self, session_id: str) -> None: + """Delete a session snapshot.""" + self._datastore.delete(_SESSIONS_COLLECTION, where={_ID_PROPERTY: session_id}) + + def _create(self, collection_name: str, record_id: str, data: str) -> None: + """Create one serialized datastore record.""" + self._datastore.create( + collection_name, + {_ID_PROPERTY: record_id, _DATA_PROPERTY: data}, + ) + + def _update(self, collection_name: str, record_id: str, data: str) -> None: + """Update one serialized datastore record.""" + updated = self._datastore.update( + collection_name, + where={_ID_PROPERTY: record_id}, + update={_DATA_PROPERTY: data}, + ) + if not updated: + raise KeyError(f"Record '{record_id}' was not found.") + + def _get_data(self, collection_name: str, record_id: str) -> str: + """Retrieve the serialized payload for one datastore record.""" + records = self._datastore.list( + collection_name, + where={_ID_PROPERTY: record_id}, + limit=1, + ) + if not records: + raise KeyError(f"Record '{record_id}' was not found.") + data = records[0].get(_DATA_PROPERTY) + if not isinstance(data, str): + raise TypeError(f"Record '{record_id}' contains invalid serialized data.") + return data diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/_utils.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/_utils.py index 27f89ee56..df5ca5a3d 100644 --- a/wayflowcore/src/wayflowcore/tools/codeexecutors/_utils.py +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/_utils.py @@ -6,17 +6,9 @@ from dataclasses import dataclass, field -from typing import Any, Literal +from typing import Any - -class NotGiven: - """Marker for a value that was not supplied.""" - - def __bool__(self) -> Literal[False]: - return False - - -NOT_GIVEN = NotGiven() +from wayflowcore._utils.notgiven import NOT_GIVEN, NotGiven @dataclass(frozen=True, kw_only=True) diff --git a/wayflowcore/tests/codeserver/conftest.py b/wayflowcore/tests/codeserver/conftest.py index 43cfbb49b..256f53e5a 100644 --- a/wayflowcore/tests/codeserver/conftest.py +++ b/wayflowcore/tests/codeserver/conftest.py @@ -14,3 +14,11 @@ def python_service() -> CodeExecutionService: """Provides a service backed by the local Python implementation.""" return CodeExecutionService(backend=LocalPythonBackend()) + + +@pytest.fixture +def python_backend() -> LocalPythonBackend: + """Provides a local Python backend for direct backend tests.""" + backend = LocalPythonBackend() + yield backend + backend.close_all_sessions() diff --git a/wayflowcore/tests/codeserver/test_app.py b/wayflowcore/tests/codeserver/test_app.py new file mode 100644 index 000000000..cd4f43b96 --- /dev/null +++ b/wayflowcore/tests/codeserver/test_app.py @@ -0,0 +1,79 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""HTTP tests for the Code Executor Protocol application.""" + +from fastapi.testclient import TestClient + +from wayflowcore.codeserver.backends.local_python import LocalPythonBackend +from wayflowcore.codeserver.server import CodeExecutorServer + + +def _client() -> TestClient: + """Create a test client for the local Python server.""" + return TestClient(CodeExecutorServer(backend=LocalPythonBackend()).get_app()) + + +def test_code_executor_capabilities() -> None: + """Returns server and local-backend capabilities.""" + response = _client().get("/v1/code-executor") + + assert response.status_code == 200 + assert response.json()["view"] == "public" + assert response.json()["server_name"] == "wayflow-code-server" + assert response.json()["capabilities"]["supported_languages"] == ["python"] + + +def test_code_executor_runs_script_to_completion() -> None: + """Runs a script through the HTTP endpoint.""" + response = _client().post( + "/v1/executions", + json={ + "language_id": "python", + "input": [{"type": "script", "source_code": "print('hello')"}], + "wait": True, + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["status"] == "completed" + assert body["output"][0]["content"][0]["text"] == "hello\n" + + +def test_code_executor_supports_polling_and_cancellation() -> None: + """Creates a pending execution and cancels it through HTTP.""" + client = _client() + response = client.post( + "/v1/executions", + json={ + "language_id": "python", + "input": [{"type": "script", "source_code": "while True: pass"}], + "wait": False, + }, + ) + execution_id = response.json()["id"] + + assert response.status_code == 200 + assert response.json()["status"] == "working" + cancellation = client.post(f"/v1/executions/{execution_id}/cancel") + + assert cancellation.status_code == 200 + assert cancellation.json()["status"] == "cancelled" + + +def test_code_executor_returns_not_found_for_unknown_execution() -> None: + """Returns HTTP 404 for an unknown execution identifier.""" + response = _client().get("/v1/executions/exec_unknown") + + assert response.status_code == 404 + + +def test_code_executor_does_not_expose_session_routes() -> None: + """Leaves session routes unavailable while session HTTP support is deferred.""" + response = _client().post("/v1/sessions", json={"language_id": "python"}) + + assert response.status_code == 404 diff --git a/wayflowcore/tests/codeserver/test_local_python_backend.py b/wayflowcore/tests/codeserver/test_local_python_backend.py new file mode 100644 index 000000000..f09b08ee5 --- /dev/null +++ b/wayflowcore/tests/codeserver/test_local_python_backend.py @@ -0,0 +1,312 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for the local Python execution backend.""" + +import time + +from wayflowcore.codeserver.backend import ( + BackendExecutionResult, + BackendHostCallbackRequest, +) +from wayflowcore.codeserver.backends.local_python import LocalPythonBackend +from wayflowcore.codeserver.models import ( + TASK_STATUS_CANCELLED, + TASK_STATUS_COMPLETED, + TASK_STATUS_FAILED, + TASK_STATUS_INPUT_REQUIRED, + TASK_STATUS_TIMED_OUT, + HostCallbackResponse, + HostInteractions, +) +from wayflowcore.codeserver.sessions import BackendSession + + +class _JavascriptSession(BackendSession): + """Minimal non-Python session used to test backend language validation.""" + + session_id = "session-1" + language_id = "javascript" + host_interactions = None + + @property + def is_active(self) -> bool: + """Return whether this test session is active.""" + return True + + def close(self) -> None: + """Implement the backend session resource interface for this test.""" + + +def test_local_python_backend_runs_script(python_backend: LocalPythonBackend) -> None: + """Runs a script and returns its normalized backend result.""" + execution = python_backend.start_script("print('hello from the backend')") + + result = execution.wait() + + assert result == BackendExecutionResult( + status=TASK_STATUS_COMPLETED, + stdout="hello from the backend\n", + ) + + +def test_local_python_backend_runs_function(python_backend: LocalPythonBackend) -> None: + """Runs a named function with JSON-compatible arguments.""" + execution = python_backend.start_function( + "def multiply(a, b):\n return a * b", + "multiply", + {"a": 6, "b": 7}, + ) + + assert execution.wait() == BackendExecutionResult( + status=TASK_STATUS_COMPLETED, + structured_content=42, + ) + + +def test_local_python_backend_captures_stdout_and_stderr( + python_backend: LocalPythonBackend, +) -> None: + """Captures standard output and standard error independently.""" + execution = python_backend.start_script( + "import sys\nprint('out')\nprint('err', file=sys.stderr)" + ) + + assert execution.wait() == BackendExecutionResult( + status=TASK_STATUS_COMPLETED, + stdout="out\n", + stderr="err\n", + ) + + +def test_local_python_backend_preserves_json_null_function_result( + python_backend: LocalPythonBackend, +) -> None: + """Preserves ``None`` as a structured JSON null result.""" + execution = python_backend.start_function( + "def produce_null():\n return None", + "produce_null", + {}, + ) + + result = execution.wait() + + assert result.status == TASK_STATUS_COMPLETED + assert result.structured_content is None + + +def test_local_python_backend_isolates_stateless_executions( + python_backend: LocalPythonBackend, +) -> None: + """Does not share variables between stateless executions.""" + first = python_backend.start_script("secret = 42") + assert first.wait().status == TASK_STATUS_COMPLETED + + second = python_backend.start_script("print(secret)") + + assert second.wait().status == TASK_STATUS_FAILED + + +def test_local_python_backend_rejects_unknown_function( + python_backend: LocalPythonBackend, +) -> None: + """Reports failure when the requested function is not defined.""" + execution = python_backend.start_function("value = 1", "missing", {}) + + assert execution.wait().status == TASK_STATUS_FAILED + + +def test_local_python_backend_returns_failed_result_for_exception( + python_backend: LocalPythonBackend, +) -> None: + """Reports a function exception as a failed result.""" + execution = python_backend.start_function( + "def fail():\n raise ValueError('boom')", + "fail", + {}, + ) + + result = execution.wait() + + assert result.status == TASK_STATUS_FAILED + assert result.error is not None + + +def test_local_python_backend_returns_failed_result_for_non_json_result( + python_backend: LocalPythonBackend, +) -> None: + """Rejects a function result that cannot be represented as JSON.""" + execution = python_backend.start_function( + "def produce_object():\n return object()", + "produce_object", + {}, + ) + + assert execution.wait().status == TASK_STATUS_FAILED + + +def test_local_python_backend_returns_timed_out_result( + python_backend: LocalPythonBackend, +) -> None: + """Stops an execution exceeding the configured timeout.""" + python_backend.execution_timeout_seconds = 0.05 + execution = python_backend.start_script("while True: pass") + + assert execution.wait().status == TASK_STATUS_TIMED_OUT + + +def test_local_python_backend_cancels_execution(python_backend: LocalPythonBackend) -> None: + """Terminates an active execution when cancellation is requested.""" + execution = python_backend.start_script("while True: pass") + time.sleep(0.01) + + assert execution.cancel().status == TASK_STATUS_CANCELLED + + +def test_local_python_backend_terminates_worker_after_timeout( + python_backend: LocalPythonBackend, +) -> None: + """Terminates the worker instead of leaving timed-out code running.""" + python_backend.execution_timeout_seconds = 0.05 + execution = python_backend.start_script("while True: pass") + + assert execution.wait().status == TASK_STATUS_TIMED_OUT + assert execution.is_alive() is False + + +def test_local_python_backend_terminates_worker_after_cancellation( + python_backend: LocalPythonBackend, +) -> None: + """Terminates the worker process after cancellation.""" + execution = python_backend.start_script("while True: pass") + time.sleep(0.01) + + assert execution.cancel().status == TASK_STATUS_CANCELLED + assert execution.is_alive() is False + + +def test_local_python_backend_reuses_session_namespace( + python_backend: LocalPythonBackend, +) -> None: + """Preserves state across executions in one session.""" + session = python_backend.create_session("session-1", "python") + assert ( + python_backend.start_script("value = 42", session=session).wait().status + == TASK_STATUS_COMPLETED + ) + + result = python_backend.start_script("print(value)", session=session).wait() + + assert result.stdout == "42\n" + + +def test_local_python_backend_does_not_share_state_between_sessions( + python_backend: LocalPythonBackend, +) -> None: + """Keeps namespaces isolated between sessions.""" + first = python_backend.create_session("session-1", "python") + second = python_backend.create_session("session-2", "python") + python_backend.start_script("value = 42", session=first).wait() + + result = python_backend.start_script("print(value)", session=second).wait() + + assert result.status == TASK_STATUS_FAILED + + +def test_local_python_backend_rejects_language_mismatch( + python_backend: LocalPythonBackend, +) -> None: + """Rejects a session whose language does not match the backend request.""" + session = _JavascriptSession() + + execution = python_backend.start_script("print('hello')", session=session) + + assert execution.wait().status == TASK_STATUS_FAILED + + +def test_local_python_backend_returns_callback_host_request( + python_backend: LocalPythonBackend, +) -> None: + """Returns a host callback when Python invokes a host function. + + The syntax used to create a callback request may be backend-dependent. + """ + session = python_backend.create_session( + "session-1", + "python", + host_interactions=HostInteractions( + enabled=True, + allowed_request_types=["tool_execution"], + ), + ) + execution = python_backend.start_script( + "result = host.tool_execution('lookup_weather', city='Paris')\nprint(result)", + session=session, + ) + + result = execution.wait() + + assert result.status == TASK_STATUS_INPUT_REQUIRED + assert result.host_callback_request == BackendHostCallbackRequest( + request_id=result.host_callback_request.request_id, # type: ignore[union-attr] + request_type="tool_execution", + name="lookup_weather", + arguments={"city": "Paris"}, + ) + + +def test_local_python_backend_resumes_after_callback_response( + python_backend: LocalPythonBackend, +) -> None: + """Resumes a paused session execution after a host callback response.""" + session = python_backend.create_session( + "session-1", + "python", + host_interactions=HostInteractions( + enabled=True, + allowed_request_types=["tool_execution"], + ), + ) + execution = python_backend.start_script( + "result = host.tool_execution('lookup_weather', city='Paris')\nprint(result)", + session=session, + ) + request = execution.wait() + callback_request = request.host_callback_request + assert callback_request is not None + + response = HostCallbackResponse( + type="host_response", + request_id=callback_request.request_id, + result={"temperature": 21}, + ) + + assert python_backend.resume_callback(session, response).wait().status == TASK_STATUS_COMPLETED + + +def test_local_python_backend_rejects_unknown_callback_request_id( + python_backend: LocalPythonBackend, +) -> None: + """Rejects a callback response that does not match a pending request.""" + session = python_backend.create_session( + "session-1", + "python", + host_interactions=HostInteractions( + enabled=True, + allowed_request_types=["tool_execution"], + ), + ) + + execution = python_backend.resume_callback( + session, + HostCallbackResponse( + type="host_response", + request_id="unknown", + result={"temperature": 21}, + ), + ) + + assert execution.wait().status == TASK_STATUS_FAILED diff --git a/wayflowcore/tests/codeserver/test_models.py b/wayflowcore/tests/codeserver/test_models.py index 9eb33fb39..8c6f65325 100644 --- a/wayflowcore/tests/codeserver/test_models.py +++ b/wayflowcore/tests/codeserver/test_models.py @@ -10,13 +10,13 @@ from wayflowcore.codeserver.models import ( CodeExecutionRequest, + CreateSessionRequest, ExecutionResponse, ExecutionResult, FunctionInput, HostCallbackRequest, - HostResponse, + HostCallbackResponse, ScriptInput, - SessionRequest, SessionSnapshot, TextContent, ) @@ -85,7 +85,7 @@ def test_execution_request_raises_on_unknown_input_type() -> None: def test_execution_request_raises_on_multiple_input_items() -> None: - """Raises when more than one executable input item is supplied in v1.""" + """Raises when more than one executable input item is supplied.""" with pytest.raises(ValueError): CodeExecutionRequest.model_validate( { @@ -187,7 +187,7 @@ def test_host_request_and_host_response_models() -> None: "arguments": {"city": "Paris"}, } ) - response = HostResponse.model_validate( + response = HostCallbackResponse.model_validate( { "type": "host_response", "request_id": "req_123", @@ -203,10 +203,9 @@ def test_host_request_and_host_response_models() -> None: def test_session_request_and_snapshot_models() -> None: """Validates session creation and lifecycle snapshot models.""" - request = SessionRequest.model_validate( + request = CreateSessionRequest.model_validate( { "language_id": "python", - "language_version": "3.12", "host_interactions": { "enabled": True, "allowed_request_types": ["tool_execution"], @@ -220,7 +219,6 @@ def test_session_request_and_snapshot_models() -> None: "object": "session", "status": "active", "language_id": "python", - "language_version": "3.12", "host_interactions": { "enabled": True, "allowed_request_types": ["tool_execution"], diff --git a/wayflowcore/tests/codeserver/test_service_core.py b/wayflowcore/tests/codeserver/test_service_core.py index a715f6ae2..affe07d33 100644 --- a/wayflowcore/tests/codeserver/test_service_core.py +++ b/wayflowcore/tests/codeserver/test_service_core.py @@ -111,7 +111,7 @@ def test_service_preserves_json_null_function_result(python_service: CodeExecuti wait=True, ) - response = service.execute(request) + response = python_service.execute(request) output = response.output[0] assert isinstance(output, ExecutionResult) @@ -132,7 +132,7 @@ def test_service_captures_stdout_and_stderr(python_service: CodeExecutionService wait=True, ) - response = service.execute(request) + response = python_service.execute(request) assert response.output == [ ExecutionResult( diff --git a/wayflowcore/tests/codeserver/test_service_host_interactions.py b/wayflowcore/tests/codeserver/test_service_host_interactions.py index 36a6a48a9..30c3e3e1f 100644 --- a/wayflowcore/tests/codeserver/test_service_host_interactions.py +++ b/wayflowcore/tests/codeserver/test_service_host_interactions.py @@ -26,10 +26,9 @@ def test_service_with_python_backend_returns_callback_host_request( """Returns a callback host request when Python code invokes a host function.""" session = python_service.create_session(CreateSessionRequest(language_id="python")) source_code = """ -result = lookup_weather(city="Paris") +result = host.tool_execution("lookup_weather", city="Paris") print(result) """ - # The syntax used to create a callback host request may be backend-dependent. request = CodeExecutionRequest( language_id="python", session_id=session.id, @@ -56,7 +55,7 @@ def test_service_waits_for_callback_host_response( input=[ ScriptInput( type="script", - source_code='result = lookup_weather(city="Paris")', + source_code='result = host.tool_execution("lookup_weather", city="Paris")', ) ], ) @@ -80,7 +79,9 @@ def test_service_resumes_execution_after_callback_host_response( input=[ ScriptInput( type="script", - source_code=('result = lookup_weather(city="Paris")\n' "print(result)"), + source_code=( + 'result = host.tool_execution("lookup_weather", city="Paris")\n' "print(result)" + ), ) ], ) From 9218055fd55fac43faf5de793c0de16c3fa450bb Mon Sep 17 00:00:00 2001 From: Paul Cayet Date: Thu, 16 Jul 2026 08:39:25 +0000 Subject: [PATCH 3/6] Add session service and HTTP routes --- wayflowcore/src/wayflowcore/codeserver/app.py | 20 ++ .../src/wayflowcore/codeserver/models.py | 12 +- .../src/wayflowcore/codeserver/service.py | 65 ++++- wayflowcore/tests/codeserver/test_app.py | 233 ++++++++++++++++-- .../test_service_host_interactions.py | 37 ++- 5 files changed, 339 insertions(+), 28 deletions(-) diff --git a/wayflowcore/src/wayflowcore/codeserver/app.py b/wayflowcore/src/wayflowcore/codeserver/app.py index f49fbf577..4d59e82ce 100644 --- a/wayflowcore/src/wayflowcore/codeserver/app.py +++ b/wayflowcore/src/wayflowcore/codeserver/app.py @@ -13,7 +13,9 @@ from wayflowcore.codeserver.models import ( CodeExecutionRequest, CodeExecutorCapabilities, + CreateSessionRequest, ExecutionResponse, + SessionSnapshot, ) from wayflowcore.codeserver.service import CodeExecutionService @@ -69,4 +71,22 @@ def cancel_execution(execution_id: str) -> ExecutionResponse: except KeyError as exc: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + @app.post("/v1/sessions", response_model=SessionSnapshot) + def create_session(request: CreateSessionRequest) -> SessionSnapshot: + """Create a stateful execution session.""" + try: + return service.create_session(request) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + @app.delete("/v1/sessions/{session_id}", response_model=SessionSnapshot) + def close_session(session_id: str) -> SessionSnapshot: + """Close a stateful execution session.""" + try: + return service.close_session(session_id) + except KeyError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + return app diff --git a/wayflowcore/src/wayflowcore/codeserver/models.py b/wayflowcore/src/wayflowcore/codeserver/models.py index 93c1619a7..81864e572 100644 --- a/wayflowcore/src/wayflowcore/codeserver/models.py +++ b/wayflowcore/src/wayflowcore/codeserver/models.py @@ -14,6 +14,8 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from typing_extensions import TypeAliasType +from wayflowcore._utils.notgiven import NOT_GIVEN, NotGiven + JsonValue = TypeAliasType( "JsonValue", "None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue]", @@ -100,12 +102,18 @@ def _validate_input_count(self) -> CodeExecutionRequest: class ExecutionResult(CodeExecutorModel): """Output item returned by an execution.""" + model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) + type: Literal["output"] content: list[TextContent] = Field(default_factory=list) """Content blocks such as captured text output.""" - structured_content: JsonValue = Field(default=None, alias="structuredContent") - """Optional JSON-compatible structured result, including scalar values and null.""" + structured_content: JsonValue | NotGiven = Field( + default=NOT_GIVEN, + alias="structuredContent", + exclude_if=lambda value: value is NOT_GIVEN, + ) + """Optional JSON-compatible structured result, including an explicit null.""" is_error: bool = Field(default=False, alias="isError") """Whether the output describes an execution error.""" diff --git a/wayflowcore/src/wayflowcore/codeserver/service.py b/wayflowcore/src/wayflowcore/codeserver/service.py index 20ed4ad78..a006c6fc4 100644 --- a/wayflowcore/src/wayflowcore/codeserver/service.py +++ b/wayflowcore/src/wayflowcore/codeserver/service.py @@ -36,6 +36,7 @@ SessionSnapshot, TextContent, ) +from wayflowcore.codeserver.sessions import BackendSession from wayflowcore.codeserver.storage import CodeExecutorStorage @@ -57,6 +58,7 @@ def __init__( self.backend = backend self.storage = storage or CodeExecutorStorage() self._execution_contexts: dict[str, BackendExecutionContext] = {} + self._sessions: dict[str, BackendSession] = {} def execute(self, request: CodeExecutionRequest) -> ExecutionResponse: """Create an execution and optionally wait for its completion. @@ -77,6 +79,8 @@ def execute(self, request: CodeExecutionRequest) -> ExecutionResponse: initial_response = self.create_execution(request) context = self._execution_contexts[initial_response.id] result = context.wait() + if isinstance(request.input[0], HostCallbackResponse) and result.error is not None: + raise ValueError(f"host request rejected: {result.error}") return self._update_execution(initial_response, result) def create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: @@ -108,20 +112,28 @@ def create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: def _start_execution(self, request: CodeExecutionRequest) -> BackendExecutionContext: """Start one stateless script or function request on the backend.""" self.backend.validate_language(request.language_id) + session = None if request.session_id is not None: - raise NotImplementedError("Session execution is not implemented yet.") + session = self._get_backend_session(request.session_id) + if request.language_id != session.language_id: + raise ValueError("Execution language does not match session language.") + if not session.is_active: + raise ValueError("Session is closed.") input_item = request.input[0] if isinstance(input_item, ScriptInput): - return self.backend.start_script(input_item.source_code) + return self.backend.start_script(input_item.source_code, session=session) if isinstance(input_item, FunctionInput): return self.backend.start_function( input_item.source_code, input_item.function_name, input_item.arguments, + session=session, ) if isinstance(input_item, HostCallbackResponse): - raise ValueError("Host callback responses require a session.") + if session is None: + raise ValueError("Host callback responses require a session.") + return self.backend.resume_callback(session, input_item) raise TypeError(f"Unsupported execution input: {type(input_item).__name__}") def _update_execution( @@ -269,7 +281,23 @@ def create_session(self, request: CreateSessionRequest) -> SessionSnapshot: SessionSnapshot Initial active session snapshot. """ - raise NotImplementedError + self.backend.validate_language(request.language_id) + session_id = self._create_session_id() + backend_session = self.backend.create_session( + session_id, + request.language_id, + host_interactions=request.host_interactions, + ) + self._sessions[session_id] = backend_session + snapshot = SessionSnapshot( + id=session_id, + object="session", + status="active", + language_id=request.language_id, + host_interactions=request.host_interactions, + metadata=request.metadata, + ) + return self.storage.create_session(snapshot) def get_session(self, session_id: str) -> SessionSnapshot: """Return the latest snapshot for a session. @@ -284,7 +312,7 @@ def get_session(self, session_id: str) -> SessionSnapshot: SessionSnapshot Latest session snapshot. """ - raise NotImplementedError + return self.storage.get_session(session_id) def close_session(self, session_id: str) -> SessionSnapshot: """Close a stateful execution session. @@ -299,4 +327,29 @@ def close_session(self, session_id: str) -> SessionSnapshot: SessionSnapshot Session snapshot after closure is requested. """ - raise NotImplementedError + snapshot = self.storage.get_session(session_id) + if snapshot.status == "closed": + return snapshot + backend_session = self._get_backend_session(session_id) + self.backend.close_session(backend_session) + self._sessions.pop(session_id, None) + closed = snapshot.model_copy(update={"status": "closed"}) + return self.storage.update_session(closed) + + @staticmethod + def _create_session_id() -> str: + """Create one public session identifier.""" + return f"sess_{uuid4().hex}" + + def _get_backend_session(self, session_id: str) -> BackendSession: + """Return the backend session associated with a public identifier.""" + try: + return self._sessions[session_id] + except KeyError as exc: + try: + snapshot = self.storage.get_session(session_id) + except KeyError: + raise ValueError(f"Unknown session id: {session_id}") from exc + if snapshot.status == "closed": + raise ValueError("Session is closed.") from exc + raise ValueError(f"Unknown session id: {session_id}") from exc diff --git a/wayflowcore/tests/codeserver/test_app.py b/wayflowcore/tests/codeserver/test_app.py index cd4f43b96..889676ad1 100644 --- a/wayflowcore/tests/codeserver/test_app.py +++ b/wayflowcore/tests/codeserver/test_app.py @@ -6,20 +6,27 @@ """HTTP tests for the Code Executor Protocol application.""" +import time + +import pytest from fastapi.testclient import TestClient from wayflowcore.codeserver.backends.local_python import LocalPythonBackend from wayflowcore.codeserver.server import CodeExecutorServer -def _client() -> TestClient: - """Create a test client for the local Python server.""" - return TestClient(CodeExecutorServer(backend=LocalPythonBackend()).get_app()) +@pytest.fixture +def client() -> TestClient: + """Create an HTTP client backed by a local Python server.""" + backend = LocalPythonBackend() + with TestClient(CodeExecutorServer(backend=backend).get_app()) as test_client: + yield test_client + backend.close_all_sessions() -def test_code_executor_capabilities() -> None: +def test_code_executor_capabilities(client: TestClient) -> None: """Returns server and local-backend capabilities.""" - response = _client().get("/v1/code-executor") + response = client.get("/v1/code-executor") assert response.status_code == 200 assert response.json()["view"] == "public" @@ -27,9 +34,9 @@ def test_code_executor_capabilities() -> None: assert response.json()["capabilities"]["supported_languages"] == ["python"] -def test_code_executor_runs_script_to_completion() -> None: +def test_code_executor_runs_script_to_completion(client: TestClient) -> None: """Runs a script through the HTTP endpoint.""" - response = _client().post( + response = client.post( "/v1/executions", json={ "language_id": "python", @@ -44,14 +51,111 @@ def test_code_executor_runs_script_to_completion() -> None: assert body["output"][0]["content"][0]["text"] == "hello\n" -def test_code_executor_supports_polling_and_cancellation() -> None: - """Creates a pending execution and cancels it through HTTP.""" - client = _client() +def test_code_executor_serializes_script_response_exactly(client: TestClient) -> None: + """Serializes captured script output with the protocol field names.""" response = client.post( "/v1/executions", json={ "language_id": "python", - "input": [{"type": "script", "source_code": "while True: pass"}], + "input": [{"type": "script", "source_code": "print('hello')"}], + "wait": True, + }, + ) + + body = response.json() + assert set(body) == { + "id", + "object", + "created_at", + "status", + "completed_at", + "language_id", + "output", + "metadata", + } + assert body["object"] == "response" + assert body["output"] == [ + { + "type": "output", + "content": [{"type": "text", "stream": "stdout", "text": "hello\n"}], + "isError": False, + } + ] + + +def test_code_executor_serializes_explicit_json_null_result(client: TestClient) -> None: + """Preserves an explicit null structured function result on the wire.""" + response = client.post( + "/v1/executions", + json={ + "language_id": "python", + "input": [ + { + "type": "function", + "source_code": "def make_none():\n return None", + "function_name": "make_none", + "arguments": {}, + } + ], + "wait": True, + }, + ) + + output = response.json()["output"][0] + assert "structuredContent" in output + assert output["structuredContent"] is None + + +def test_code_executor_runs_function_to_completion(client: TestClient) -> None: + """Runs a named function and returns its structured result.""" + response = client.post( + "/v1/executions", + json={ + "language_id": "python", + "input": [ + { + "type": "function", + "source_code": "def multiply(a, b):\n return a * b", + "function_name": "multiply", + "arguments": {"a": 6, "b": 7}, + } + ], + "wait": True, + }, + ) + + assert response.status_code == 200 + assert response.json()["output"][0]["structuredContent"] == 42 + + +def test_code_executor_returns_failed_execution(client: TestClient) -> None: + """Returns a failed response when user code raises an exception.""" + response = client.post( + "/v1/executions", + json={ + "language_id": "python", + "input": [{"type": "script", "source_code": "raise ValueError('boom')"}], + "wait": True, + }, + ) + + assert response.status_code == 200 + assert response.json()["status"] == "failed" + assert response.json()["output"][0]["isError"] is True + + +def test_code_executor_supports_polling(client: TestClient) -> None: + """Creates an execution without waiting and retrieves its snapshot.""" + response = client.post( + "/v1/executions", + json={ + "language_id": "python", + "input": [ + { + "type": "script", + "source_code": "import time\ntime.sleep(0.1)\nprint('done')", + } + ], "wait": False, }, ) @@ -59,21 +163,118 @@ def test_code_executor_supports_polling_and_cancellation() -> None: assert response.status_code == 200 assert response.json()["status"] == "working" + for _ in range(100): + snapshot = client.get(f"/v1/executions/{execution_id}") + if snapshot.json()["status"] == "completed": + break + time.sleep(0.05) + + assert snapshot.status_code == 200 + assert snapshot.json()["status"] == "completed" + + +def test_code_executor_supports_cancellation(client: TestClient) -> None: + """Cancels a running execution through HTTP.""" + response = client.post( + "/v1/executions", + json={ + "language_id": "python", + "input": [{"type": "script", "source_code": "while True: pass"}], + "wait": False, + }, + ) + execution_id = response.json()["id"] cancellation = client.post(f"/v1/executions/{execution_id}/cancel") assert cancellation.status_code == 200 assert cancellation.json()["status"] == "cancelled" -def test_code_executor_returns_not_found_for_unknown_execution() -> None: +def test_code_executor_returns_not_found_for_unknown_execution(client: TestClient) -> None: """Returns HTTP 404 for an unknown execution identifier.""" - response = _client().get("/v1/executions/exec_unknown") + response = client.get("/v1/executions/exec_unknown") assert response.status_code == 404 -def test_code_executor_does_not_expose_session_routes() -> None: - """Leaves session routes unavailable while session HTTP support is deferred.""" - response = _client().post("/v1/sessions", json={"language_id": "python"}) +def test_code_executor_returns_not_found_when_cancelling_unknown_execution( + client: TestClient, +) -> None: + """Returns HTTP 404 when cancelling an unknown execution identifier.""" + response = client.post("/v1/executions/exec_unknown/cancel") assert response.status_code == 404 + + +def test_code_executor_rejects_malformed_request(client: TestClient) -> None: + """Returns HTTP 422 when the request does not match the protocol model.""" + response = client.post( + "/v1/executions", + json={"language_id": "python", "input": [{"type": "script"}]}, + ) + + assert response.status_code == 422 + + +def test_code_executor_rejects_unknown_request_fields(client: TestClient) -> None: + """Rejects fields that are not part of the execution request model.""" + response = client.post( + "/v1/executions", + json={ + "language_id": "python", + "input": [{"type": "script", "source_code": "pass"}], + "unexpected": True, + }, + ) + + assert response.status_code == 422 + + +def test_code_executor_rejects_unsupported_language(client: TestClient) -> None: + """Returns a client error when the backend cannot run the language.""" + response = client.post( + "/v1/executions", + json={ + "language_id": "javascript", + "input": [{"type": "script", "source_code": "console.log('hello')"}], + }, + ) + + assert response.status_code == 400 + assert isinstance(response.json()["detail"], str) + + +def test_code_executor_creates_session(client: TestClient) -> None: + """Creates a stateful session through HTTP.""" + response = client.post("/v1/sessions", json={"language_id": "python"}) + + assert response.status_code == 200 + body = response.json() + assert body["object"] == "session" + assert body["status"] == "active" + assert body["language_id"] == "python" + + +def test_code_executor_closes_session(client: TestClient) -> None: + """Closes a stateful session through HTTP.""" + created = client.post("/v1/sessions", json={"language_id": "python"}) + session_id = created.json()["id"] + + response = client.delete(f"/v1/sessions/{session_id}") + + assert response.status_code == 200 + assert response.json()["status"] == "closed" + + +def test_code_executor_returns_not_found_for_unknown_session(client: TestClient) -> None: + """Returns HTTP 404 when closing an unknown session identifier.""" + response = client.delete("/v1/sessions/sess_unknown") + + assert response.status_code == 404 + + +def test_code_executor_rejects_malformed_session_request(client: TestClient) -> None: + """Returns HTTP 422 when a session request is missing its language.""" + response = client.post("/v1/sessions", json={}) + + assert response.status_code == 422 diff --git a/wayflowcore/tests/codeserver/test_service_host_interactions.py b/wayflowcore/tests/codeserver/test_service_host_interactions.py index 30c3e3e1f..96c1c3845 100644 --- a/wayflowcore/tests/codeserver/test_service_host_interactions.py +++ b/wayflowcore/tests/codeserver/test_service_host_interactions.py @@ -15,6 +15,7 @@ CreateSessionRequest, HostCallbackRequest, HostCallbackResponse, + HostInteractions, ScriptInput, ) from wayflowcore.codeserver.service import CodeExecutionService @@ -24,7 +25,14 @@ def test_service_with_python_backend_returns_callback_host_request( python_service: CodeExecutionService, ) -> None: """Returns a callback host request when Python code invokes a host function.""" - session = python_service.create_session(CreateSessionRequest(language_id="python")) + session = python_service.create_session( + CreateSessionRequest( + language_id="python", + host_interactions=HostInteractions( + enabled=True, allowed_request_types=["tool_execution"] + ), + ) + ) source_code = """ result = host.tool_execution("lookup_weather", city="Paris") print(result) @@ -48,7 +56,14 @@ def test_service_waits_for_callback_host_response( python_service: CodeExecutionService, ) -> None: """Leaves an execution waiting while a callback host request is pending.""" - session = python_service.create_session(CreateSessionRequest(language_id="python")) + session = python_service.create_session( + CreateSessionRequest( + language_id="python", + host_interactions=HostInteractions( + enabled=True, allowed_request_types=["tool_execution"] + ), + ) + ) request = CodeExecutionRequest( language_id="python", session_id=session.id, @@ -72,7 +87,14 @@ def test_service_resumes_execution_after_callback_host_response( python_service: CodeExecutionService, ) -> None: """Resumes a callback execution with a response in the same session.""" - session = python_service.create_session(CreateSessionRequest(language_id="python")) + session = python_service.create_session( + CreateSessionRequest( + language_id="python", + host_interactions=HostInteractions( + enabled=True, allowed_request_types=["tool_execution"] + ), + ) + ) request = CodeExecutionRequest( language_id="python", session_id=session.id, @@ -111,7 +133,14 @@ def test_service_rejects_unknown_callback_host_request_id( python_service: CodeExecutionService, ) -> None: """Rejects a callback response without a matching pending host request.""" - session = python_service.create_session(CreateSessionRequest(language_id="python")) + session = python_service.create_session( + CreateSessionRequest( + language_id="python", + host_interactions=HostInteractions( + enabled=True, allowed_request_types=["tool_execution"] + ), + ) + ) request = CodeExecutionRequest( language_id="python", session_id=session.id, From 6ecc2663cc1e1841370e69d38f3bcd2c4b0b9eae Mon Sep 17 00:00:00 2001 From: Paul Cayet Date: Thu, 16 Jul 2026 09:19:05 +0000 Subject: [PATCH 4/6] add python policy support --- .../src/wayflowcore/codeserver/backend.py | 16 -- .../codeserver/backends/local_python.py | 2 +- .../backends/pythonexecutionpolicy.py | 196 ++++++++++++++++++ .../codeserver/backends/pythonworker.py | 2 +- .../test_python_execution_policy.py | 151 ++++++++++++++ 5 files changed, 349 insertions(+), 18 deletions(-) create mode 100644 wayflowcore/src/wayflowcore/codeserver/backends/pythonexecutionpolicy.py create mode 100644 wayflowcore/tests/codeserver/test_python_execution_policy.py diff --git a/wayflowcore/src/wayflowcore/codeserver/backend.py b/wayflowcore/src/wayflowcore/codeserver/backend.py index 3debdac0e..855daa27b 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backend.py +++ b/wayflowcore/src/wayflowcore/codeserver/backend.py @@ -142,19 +142,3 @@ def wait(self) -> BackendExecutionResult: def cancel(self) -> BackendExecutionResult: """Request cancellation and return the resulting backend state.""" raise NotImplementedError - - -class PythonExecutionPolicy: - """Policy controlling Python source execution inside a worker.""" - - def validate_script(self, source_code: str) -> None: - """Validate source code intended for script execution.""" - raise NotImplementedError - - def validate_function(self, source_code: str, function_name: str) -> None: - """Validate source code and entry point for function execution.""" - raise NotImplementedError - - def build_namespace(self) -> dict[str, object]: - """Build the initial namespace for a worker execution.""" - raise NotImplementedError diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py index bb1676484..764b27587 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py +++ b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py @@ -26,8 +26,8 @@ BackendExecutionResult, BackendHostCallbackRequest, CodeExecutorBackend, - PythonExecutionPolicy, ) +from wayflowcore.codeserver.backends.pythonexecutionpolicy import PythonExecutionPolicy from wayflowcore.codeserver.backends.pythonworker import worker_main from wayflowcore.codeserver.models import ( TASK_STATUS_CANCELLED, diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/pythonexecutionpolicy.py b/wayflowcore/src/wayflowcore/codeserver/backends/pythonexecutionpolicy.py new file mode 100644 index 000000000..23e68d361 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/backends/pythonexecutionpolicy.py @@ -0,0 +1,196 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Python source validation and worker namespace policies.""" + +from __future__ import annotations + +import ast +import builtins +from abc import ABC, abstractmethod +from collections.abc import Iterable, Mapping, Sequence + + +class PythonExecutionPolicy(ABC): + """Policy controlling Python source execution inside a worker.""" + + @abstractmethod + def validate_script(self, source_code: str) -> None: + """Validate source code intended for script execution.""" + raise NotImplementedError + + @abstractmethod + def validate_function(self, source_code: str, function_name: str) -> None: + """Validate source code and entry point for function execution.""" + raise NotImplementedError + + @abstractmethod + def build_namespace(self) -> dict[str, object]: + """Build the initial namespace for a worker execution.""" + raise NotImplementedError + + +class StrictPythonExecutionPolicy(PythonExecutionPolicy): + """Restrict Python syntax, imports, builtins, and attribute access.""" + + def __init__(self, allowed_imports: Iterable[str] = ()) -> None: + """Initialize the policy with the permitted top-level modules.""" + self.allowed_imports = tuple(allowed_imports) + self._allowed_imports = set(self.allowed_imports) + + def validate_script(self, source_code: str) -> None: + """Reject unsafe syntax in a script before execution starts.""" + tree = ast.parse(source_code) + _AstValidator(self._allowed_imports).visit(tree) + + def validate_function(self, source_code: str, function_name: str) -> None: + """Validate one named synchronous function and its body.""" + tree = ast.parse(source_code) + functions = [node for node in tree.body if isinstance(node, ast.FunctionDef)] + if len(tree.body) != 1 or len(functions) != 1: + raise ValueError("Function source must define exactly one function") + if functions[0].name != function_name: + raise ValueError(f"Function '{function_name}' is not defined") + _AstValidator(self._allowed_imports, allow_function=function_name).visit(tree) + + def build_namespace(self) -> dict[str, object]: + """Build a namespace with restricted builtins and host integration.""" + old_import = builtins.__import__ + + def limited_import( + name: str, + globals: Mapping[str, object] | None = None, + locals: Mapping[str, object] | None = None, + fromlist: Sequence[str] = (), + level: int = 0, + ) -> object: + top_level = name.split(".")[0] + if top_level not in self._allowed_imports: + raise ImportError(f"Import of module '{top_level}' is not allowed") + return old_import(name, globals, locals, fromlist, level) + + allowed = { + name: getattr(builtins, name) + for name in ( + "print", + "range", + "int", + "float", + "str", + "len", + "sum", + "min", + "max", + "abs", + "enumerate", + "list", + "dict", + "set", + "tuple", + "Exception", + "ValueError", + "TypeError", + "ZeroDivisionError", + "RuntimeError", + ) + } + allowed["__import__"] = limited_import + return {"__builtins__": allowed, "host": None} + + +class _AstValidator(ast.NodeVisitor): + """Validate the restricted Python AST used by the strict policy.""" + + forbidden_attrs = {"encode", "decode", "format", "format_map", "mro", "__subclasses__"} + + def __init__(self, allowed_imports: set[str], allow_function: str | None = None) -> None: + self.allowed_imports = allowed_imports + self.allow_function = allow_function + self.function_count = 0 + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + if self.allow_function != node.name or self.function_count != 0: + raise ValueError("Defining functions is not allowed") + self.function_count += 1 + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + raise ValueError("Async functions are not allowed") + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + raise ValueError("Defining classes is not allowed") + + def visit_Lambda(self, node: ast.Lambda) -> None: + raise ValueError("Lambdas are not allowed") + + def visit_Attribute(self, node: ast.Attribute) -> None: + if node.attr.startswith("__") or node.attr in self.forbidden_attrs: + raise ValueError(f"Attribute '{node.attr}' is not allowed") + self.generic_visit(node) + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: + if node.name is not None: + raise ValueError("Exception binding is not allowed") + self.generic_visit(node) + + def visit_Yield(self, node: ast.Yield) -> None: + raise ValueError("Yield is not allowed") + + def visit_YieldFrom(self, node: ast.YieldFrom) -> None: + raise ValueError("YieldFrom is not allowed") + + def visit_NamedExpr(self, node: ast.NamedExpr) -> None: + raise ValueError("Walrus operator is not allowed") + + def visit_ListComp(self, node: ast.ListComp) -> None: + raise ValueError("Comprehensions are not allowed") + + def visit_SetComp(self, node: ast.SetComp) -> None: + raise ValueError("Comprehensions are not allowed") + + def visit_DictComp(self, node: ast.DictComp) -> None: + raise ValueError("Comprehensions are not allowed") + + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None: + raise ValueError("Generator expressions are not allowed") + + def visit_Global(self, node: ast.Global) -> None: + raise ValueError("Global is not allowed") + + def visit_Nonlocal(self, node: ast.Nonlocal) -> None: + raise ValueError("Nonlocal is not allowed") + + def visit_Name(self, node: ast.Name) -> None: + if node.id.startswith("__"): + raise ValueError("Dunder names are not allowed") + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + if isinstance(node.func, ast.Name) and node.func.id in { + "exec", + "eval", + "compile", + "open", + "__import__", + "dir", + }: + raise ValueError(f"Calling '{node.func.id}' is not allowed") + self.generic_visit(node) + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + if alias.name.split(".")[0] not in self.allowed_imports: + raise ValueError(f"Import of module '{alias.name.split('.')[0]}' is not allowed") + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + if node.level > 0: + raise ValueError("Relative imports are not allowed") + module = node.module or "" + top_level = module.split(".")[0] + if top_level and top_level not in self.allowed_imports: + raise ValueError(f"Import from module '{module}' is not allowed") + self.generic_visit(node) diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py b/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py index 6cfea4b3b..a00190c3a 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py +++ b/wayflowcore/src/wayflowcore/codeserver/backends/pythonworker.py @@ -19,7 +19,7 @@ from typing import cast from wayflowcore._utils.notgiven import NOT_GIVEN, NotGiven -from wayflowcore.codeserver.backend import PythonExecutionPolicy +from wayflowcore.codeserver.backends.pythonexecutionpolicy import PythonExecutionPolicy from wayflowcore.codeserver.models import JsonValue WorkerCommand = dict[str, object] diff --git a/wayflowcore/tests/codeserver/test_python_execution_policy.py b/wayflowcore/tests/codeserver/test_python_execution_policy.py new file mode 100644 index 000000000..61610a015 --- /dev/null +++ b/wayflowcore/tests/codeserver/test_python_execution_policy.py @@ -0,0 +1,151 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Tests for strict Python execution through the local backend.""" + +import pytest + +from wayflowcore.codeserver.backends.local_python import LocalPythonBackend +from wayflowcore.codeserver.backends.pythonexecutionpolicy import ( + StrictPythonExecutionPolicy, +) +from wayflowcore.codeserver.models import TASK_STATUS_COMPLETED, TASK_STATUS_FAILED + + +@pytest.fixture +def strict_python_backend() -> LocalPythonBackend: + """Create a local backend using the strict Python execution policy.""" + backend = LocalPythonBackend( + policy=StrictPythonExecutionPolicy(allowed_imports=["math"]), + ) + yield backend + backend.close_all_sessions() + + +@pytest.mark.parametrize( + ("code", "expected_substring"), + [ + ("import os\nprint(os.getcwd())", "not allowed"), + ("from pathlib import Path\nprint(Path.cwd())", "not allowed"), + ("async def f():\n return 1", "not allowed"), + ("def f():\n return 1", "not allowed"), + ("class C:\n pass", "not allowed"), + ("print(__builtins__)", "not allowed"), + ("print(dir(1))", "not allowed"), + ("print((lambda x: x)(1))", "not allowed"), + ("print((1).__class__)", "not allowed"), + ("print((1).__subclasshook__)", "not allowed"), + ("print(''.__class__.mro()[1].__subclasses__())", "not allowed"), + ("print((1).__getattribute__)", "not allowed"), + ("print(eval('1 + 1'))", "not allowed"), + ("print(compile('1 + 1', '', 'eval'))", "not allowed"), + ("print([x for x in range(3)])", "not allowed"), + ("print({x: x for x in range(3)})", "not allowed"), + ("print((x for x in range(3)))", "not allowed"), + ("x = (y := 1)", "not allowed"), + ("try:\n 1 / 0\nexcept Exception as e:\n print('x')", "not allowed"), + ("def g():\n yield 1", "not allowed"), + ("print('x'.encode('utf-8'))", "not allowed"), + ("print(b'x'.decode('utf-8'))", "not allowed"), + ("print('{}'.format(1))", "not allowed"), + ("print(''.mro())", "not allowed"), + ], +) +def test_local_python_backend_rejects_unsafe_script( + strict_python_backend: LocalPythonBackend, + code: str, + expected_substring: str, +) -> None: + """Rejects unsafe script source before starting a worker.""" + with pytest.raises(ValueError, match=expected_substring): + strict_python_backend.start_script(code) + + +def test_local_python_backend_rejects_unsupported_script_import( + strict_python_backend: LocalPythonBackend, +) -> None: + """Rejects imports outside the configured allow-list.""" + with pytest.raises(ValueError, match="not allowed"): + strict_python_backend.start_script("import os") + + +def test_local_python_backend_accepts_allowed_script_import( + strict_python_backend: LocalPythonBackend, +) -> None: + """Runs a script importing an allowed top-level module.""" + execution = strict_python_backend.start_script("import math\nprint(math.sqrt(4))") + + assert execution.wait().status == TASK_STATUS_COMPLETED + + +def test_local_python_backend_rejects_relative_import( + strict_python_backend: LocalPythonBackend, +) -> None: + """Rejects relative imports before starting a worker.""" + with pytest.raises(ValueError, match="not allowed"): + strict_python_backend.start_script("from .math import sqrt") + + +def test_local_python_backend_runs_valid_function( + strict_python_backend: LocalPythonBackend, +) -> None: + """Allows one named synchronous function for function execution.""" + execution = strict_python_backend.start_function( + "def multiply(a, b):\n return a * b", + "multiply", + {"a": 6, "b": 7}, + ) + + result = execution.wait() + + assert result.status == TASK_STATUS_COMPLETED + assert result.structured_content == 42 + + +@pytest.mark.parametrize( + ("code", "function_name"), + [ + ("def multiply(a, b):\n return a * b", "missing"), + ("def a():\n pass\ndef b():\n pass", "a"), + ("async def multiply(a, b):\n return a * b", "multiply"), + ("class Multiply:\n pass", "Multiply"), + ("def multiply(a, b):\n return eval('1 + 1')", "multiply"), + ], +) +def test_local_python_backend_rejects_invalid_function( + strict_python_backend: LocalPythonBackend, + code: str, + function_name: str, +) -> None: + """Rejects missing, ambiguous, asynchronous, or unsafe functions.""" + with pytest.raises(ValueError): + strict_python_backend.start_function(code, function_name, {}) + + +def test_local_python_backend_restricts_builtins( + strict_python_backend: LocalPythonBackend, +) -> None: + """Reports failure when code uses a builtin outside the restricted set.""" + execution = strict_python_backend.start_script("print(exec)") + + result = execution.wait() + + assert result.status == TASK_STATUS_FAILED + assert result.error is not None + assert "exec" in result.error + + +def test_local_python_backend_preserves_output_truncation( + strict_python_backend: LocalPythonBackend, +) -> None: + """Preserves the backend output limit while using the strict policy.""" + strict_python_backend.max_stdout_chars = 100 + execution = strict_python_backend.start_script("print('a' * 2000)") + + result = execution.wait() + + assert result.status == TASK_STATUS_COMPLETED + assert len(result.stdout) >= 100 From 370e061e2dea30eb84327518fdfba27b41a2fdd7 Mon Sep 17 00:00:00 2001 From: Paul Cayet Date: Fri, 17 Jul 2026 12:33:29 +0000 Subject: [PATCH 5/6] Complete code executor server and client support --- .../code_examples/howto_serve_codeserver.py | 85 +++++++ .../howtoguides/howto_serve_codeserver.rst | 171 ++++++++++++++ .../src/wayflowcore/agentserver/server.py | 5 +- wayflowcore/src/wayflowcore/cli/__init__.py | 2 + wayflowcore/src/wayflowcore/cli/codeserver.py | 62 +++++ wayflowcore/src/wayflowcore/cli/serve.py | 15 +- .../src/wayflowcore/codeserver/__init__.py | 3 +- .../codeserver/backends/local_python.py | 6 +- .../src/wayflowcore/codeserver/server.py | 205 ++++++++++++++++- .../codeserver/serverstorageconfig.py | 33 +++ .../src/wayflowcore/codeserver/storage.py | 64 ++++-- .../wayflowcore/tools/codeexecutors/_http.py | 68 ++++++ .../tools/codeexecutors/endpointexecutor.py | 52 ++++- .../tools/codeexecutors/executor.py | 215 ++++++++++++++++-- .../codeexecutors/localcontainerexecutor.py | 2 +- .../tools/codeexecutors/subprocessexecutor.py | 166 +++++++++++++- .../tests/tools/codeexecutors/conftest.py | 56 +++++ .../tools/codeexecutors/start_codeserver.py | 26 +++ .../codeexecutors/test_codeexecutors_e2e.py | 152 +++++++++++++ 19 files changed, 1324 insertions(+), 64 deletions(-) create mode 100644 docs/wayflowcore/source/core/code_examples/howto_serve_codeserver.py create mode 100644 docs/wayflowcore/source/core/howtoguides/howto_serve_codeserver.rst create mode 100644 wayflowcore/src/wayflowcore/cli/codeserver.py create mode 100644 wayflowcore/src/wayflowcore/codeserver/serverstorageconfig.py create mode 100644 wayflowcore/src/wayflowcore/tools/codeexecutors/_http.py create mode 100644 wayflowcore/tests/tools/codeexecutors/conftest.py create mode 100644 wayflowcore/tests/tools/codeexecutors/start_codeserver.py create mode 100644 wayflowcore/tests/tools/codeexecutors/test_codeexecutors_e2e.py diff --git a/docs/wayflowcore/source/core/code_examples/howto_serve_codeserver.py b/docs/wayflowcore/source/core/code_examples/howto_serve_codeserver.py new file mode 100644 index 000000000..a6ebf221c --- /dev/null +++ b/docs/wayflowcore/source/core/code_examples/howto_serve_codeserver.py @@ -0,0 +1,85 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""HTTP examples for the WayFlow Code Executor server.""" + +import time + +import httpx + +BASE_URL = "http://127.0.0.1:8765" + +# .. start-##_Get_capabilities +response = httpx.get(f"{BASE_URL}/v1/code-executor") +response.raise_for_status() +print(response.json()) +# .. end-##_Get_capabilities + +# .. start-##_Run_script +response = httpx.post( + f"{BASE_URL}/v1/executions", + json={ + "language_id": "python", + "input": [ + { + "type": "script", + "source_code": "print('hello from the Code Executor server')", + } + ], + "wait": True, + }, +) +response.raise_for_status() +script_response = response.json() +print(script_response["output"]) +# .. end-##_Run_script + +# .. start-##_Run_function +response = httpx.post( + f"{BASE_URL}/v1/executions", + json={ + "language_id": "python", + "input": [ + { + "type": "function", + "source_code": "def multiply(a, b):\n return a * b", + "function_name": "multiply", + "arguments": {"a": 6, "b": 7}, + } + ], + "wait": True, + }, +) +response.raise_for_status() +function_response = response.json() +print(function_response["output"][0]["structuredContent"]) +# .. end-##_Run_function + +# .. start-##_Poll_execution +response = httpx.post( + f"{BASE_URL}/v1/executions", + json={ + "language_id": "python", + "input": [ + { + "type": "script", + "source_code": "import time\ntime.sleep(1)\nprint('done')", + } + ], + "wait": False, + }, +) +response.raise_for_status() +execution = response.json() + +while execution["status"] not in {"completed", "failed", "timed_out", "cancelled"}: + time.sleep(0.1) + response = httpx.get(f"{BASE_URL}/v1/executions/{execution['id']}") + response.raise_for_status() + execution = response.json() + +print(execution) +# .. end-##_Poll_execution diff --git a/docs/wayflowcore/source/core/howtoguides/howto_serve_codeserver.rst b/docs/wayflowcore/source/core/howtoguides/howto_serve_codeserver.rst new file mode 100644 index 000000000..f12d68475 --- /dev/null +++ b/docs/wayflowcore/source/core/howtoguides/howto_serve_codeserver.rst @@ -0,0 +1,171 @@ +.. _top-howtoservecodeserver: + +================================= +How to Run a Code Executor Server +================================= + +.. |python-icon| image:: ../../_static/icons/python-icon.svg + :width: 40px + :height: 40px + +.. grid:: 2 + + .. grid-item-card:: |python-icon| Download Python Script + :link: ../code_examples/howto_serve_codeserver.py + :link-alt: Run a Code Executor Server how-to script + + Python script for this guide. + + +WayFlow provides a Code Executor Protocol and a compatible server for running Python scripts and +functions. The server exposes a small HTTP API for checking capabilities, submitting executions, +and polling execution results. + + +Start the server +================ + +Start a local Python Code Executor server with the WayFlow CLI: + +.. code-block:: bash + + wayflow codeserver --host 127.0.0.1 --port 8765 + +The server is unauthenticated by default for local development. For deployments, put it behind +an authentication and TLS layer, and apply the resource limits appropriate for your environment. + + +Run the server in a container +============================= + +You can also build and run the local Python Code Executor server with Podman or Docker. The +following is the container definition: + +.. code-block:: Dockerfile + + ARG PYTHON_BASE_IMAGE=python:3.11-slim + FROM ${PYTHON_BASE_IMAGE} + + ENV PYTHONUNBUFFERED=1 + WORKDIR /opt/wayflow + + COPY wayflowcore /opt/wayflow/wayflowcore + COPY VERSION /opt/wayflow/VERSION + + RUN python3 -m pip install --no-cache-dir --upgrade pip \ + && python3 -m pip install --no-cache-dir -e /opt/wayflow/wayflowcore + + EXPOSE 8765 + + CMD ["wayflow", "codeserver", "--host", "0.0.0.0", "--port", "8765"] + +Build the image from the directory containing ``Containerfile.local-python-codeserver``: + +.. tabs:: + + .. tab:: Podman + + .. code-block:: bash + + podman build \ + -f Containerfile.local-python-codeserver \ + -t localhost/wayflow-code-server-local-python:dev . + + If Podman encounters SELinux labeling issues on RHEL, you may want to look into + ``--security-opt`` configuration parameters. + + .. tab:: Docker + + .. code-block:: bash + + docker build \ + -f Containerfile.local-python-codeserver \ + -t wayflow-code-server-local-python:dev . + +Run the container with an API key because it listens on all interfaces: + +.. tabs:: + + .. tab:: Podman + + .. code-block:: bash + + podman run --rm \ + --name wayflow-code-server \ + -p 8765:8765 \ + -e WAYFLOW_API_KEY='your-secret-key' \ + localhost/wayflow-code-server-local-python:dev + + .. tab:: Docker + + .. code-block:: bash + + docker run --rm \ + --name wayflow-code-server \ + -p 8765:8765 \ + -e WAYFLOW_API_KEY='your-secret-key' \ + wayflow-code-server-local-python:dev + + +Check server capabilities +========================= + +The capabilities endpoint reports the languages and execution modes supported by the server. + +.. literalinclude:: ../code_examples/howto_serve_codeserver.py + :language: python + :start-after: .. start-##_Get_capabilities + :end-before: .. end-##_Get_capabilities + + +Run a script +============ + +Submit a script and wait for it to complete. Captured output is returned in the execution result. + +.. literalinclude:: ../code_examples/howto_serve_codeserver.py + :language: python + :start-after: .. start-##_Run_script + :end-before: .. end-##_Run_script + + +Run a function +============== + +Submit source code containing one named function and pass JSON-compatible named arguments. + +.. literalinclude:: ../code_examples/howto_serve_codeserver.py + :language: python + :start-after: .. start-##_Run_function + :end-before: .. end-##_Run_function + + +Submit and poll an execution +============================ + +Set ``wait`` to ``False`` to receive an execution identifier immediately. Poll the execution +endpoint until it reaches a terminal status. + +.. literalinclude:: ../code_examples/howto_serve_codeserver.py + :language: python + :start-after: .. start-##_Poll_execution + :end-before: .. end-##_Poll_execution + + +Security considerations +======================= + +The Code Executor server executes submitted source code. Do not expose an unauthenticated server +to an untrusted network. For production deployments, add authentication, TLS, rate limiting, and +resource controls through an API gateway, reverse proxy, or deployment-specific middleware. + + +Full code +========= + +Click the card at the :ref:`top of this page ` to download the Python +example for this guide or copy the code below. + +.. literalinclude:: ../code_examples/howto_serve_codeserver.py + :language: python + :linenos: diff --git a/wayflowcore/src/wayflowcore/agentserver/server.py b/wayflowcore/src/wayflowcore/agentserver/server.py index 317a85a00..bef0d24dc 100644 --- a/wayflowcore/src/wayflowcore/agentserver/server.py +++ b/wayflowcore/src/wayflowcore/agentserver/server.py @@ -62,7 +62,7 @@ def __init__( Config for the storage to save the conversations. If not provided, the default storage with `InMemoryDatastore` will be used. """ - self.storage_config = ServerStorageConfig() + self.storage_config = storage_config or ServerStorageConfig() self._storage = A2AStorage(self.storage_config) self._broker = InMemoryBroker() @@ -179,7 +179,8 @@ def __init__( Datastore for server persistence. Needs to have the proper table and columns as specified in the `storage_config`. storage_config: - Cch will not guarantee persistence of data across runs. + Configuration for the datastore schema and retention behavior. When omitted, + the default in-memory storage configuration is used. allowed_origins: Origins allowed to make browser cross-origin requests to the server through CORS (Cross-Origin Resource Sharing). CORS is a browser access-control diff --git a/wayflowcore/src/wayflowcore/cli/__init__.py b/wayflowcore/src/wayflowcore/cli/__init__.py index e6e101a0c..825bf0472 100644 --- a/wayflowcore/src/wayflowcore/cli/__init__.py +++ b/wayflowcore/src/wayflowcore/cli/__init__.py @@ -7,6 +7,7 @@ import argparse from typing import Optional, Sequence +from .codeserver import add_parser as add_codeserver_parser from .serve import add_parser as add_serve_parser __all__ = ["main"] @@ -20,6 +21,7 @@ def build_parser() -> argparse.ArgumentParser: subparsers = parser.add_subparsers(dest="command", required=True) add_serve_parser(subparsers) + add_codeserver_parser(subparsers) return parser diff --git a/wayflowcore/src/wayflowcore/cli/codeserver.py b/wayflowcore/src/wayflowcore/cli/codeserver.py new file mode 100644 index 000000000..183a031b7 --- /dev/null +++ b/wayflowcore/src/wayflowcore/cli/codeserver.py @@ -0,0 +1,62 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Command-line entry point for the Code Executor server.""" + +from __future__ import annotations + +import argparse +import os + +from wayflowcore.codeserver import CodeExecutorServer + +__all__ = ["add_parser", "codeserver"] + + +def add_parser( + subparsers: argparse._SubParsersAction[argparse.ArgumentParser], +) -> argparse.ArgumentParser: + """Add the ``codeserver`` command to the WayFlow CLI parser.""" + parser = subparsers.add_parser( + "codeserver", + help="Run a local Python Code Executor server.", + description="Launch a Code Executor Protocol server with the local Python backend.", + ) + parser.add_argument( + "--host", + default="127.0.0.1", + help="Host interface to bind (default: 127.0.0.1).", + ) + parser.add_argument( + "--port", + default=8765, + type=int, + help="Port to bind (default: 8765).", + ) + parser.add_argument( + "--api-key", + default=os.getenv("WAYFLOW_API_KEY"), + help="Bearer token required by the server (or WAYFLOW_API_KEY).", + ) + parser.set_defaults(handler=_run_codeserver) + return parser + + +def _run_codeserver(args: argparse.Namespace) -> None: + """Run the configured local Python Code Executor server.""" + codeserver(host=args.host, port=args.port, api_key=args.api_key) + + +def codeserver(host: str = "127.0.0.1", port: int = 8765, api_key: str | None = None) -> None: + """Run a local Python Code Executor server.""" + CodeExecutorServer().run(host=host, port=port, api_key=api_key) + + +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. diff --git a/wayflowcore/src/wayflowcore/cli/serve.py b/wayflowcore/src/wayflowcore/cli/serve.py index 66108ff46..865c158a5 100644 --- a/wayflowcore/src/wayflowcore/cli/serve.py +++ b/wayflowcore/src/wayflowcore/cli/serve.py @@ -17,11 +17,6 @@ import yaml from wayflowcore.agentserver import ServerStorageConfig -from wayflowcore.agentserver._storagehelpers import ( - _prepare_oracle_datastore, - _prepare_postgres_datastore, -) -from wayflowcore.agentserver.app import create_server_app from wayflowcore.agentspec import AgentSpecLoader from wayflowcore.conversationalcomponent import ConversationalComponent from wayflowcore.datastore import ( @@ -218,6 +213,8 @@ def serve( api_key: Optional[str] = None, ) -> None: + from wayflowcore.agentserver.app import create_server_app + agents: dict[str, ConversationalComponent] = {} if len(agent_configs) != len(agent_ids): raise ValueError("You specified different numbers of agents and configs") @@ -288,6 +285,12 @@ def _get_persistence_arguments( datastore_connection_config: Optional[Any] = None, setup_datastore: bool = False, ) -> Tuple[Datastore, ServerStorageConfig]: + from wayflowcore.agentserver import ServerStorageConfig + from wayflowcore.agentserver._storagehelpers import ( + _prepare_oracle_datastore, + _prepare_postgres_datastore, + ) + storage_config = storage_config or ServerStorageConfig() storage_schema = storage_config.to_schema() storage: Datastore @@ -363,6 +366,8 @@ def _load_yaml_dict(path: Path) -> Dict[str, Any]: def _load_server_storage_config(path_str: Optional[str]) -> Optional[ServerStorageConfig]: + from wayflowcore.agentserver import ServerStorageConfig + if path_str is None: return None config_path = Path(path_str).expanduser() diff --git a/wayflowcore/src/wayflowcore/codeserver/__init__.py b/wayflowcore/src/wayflowcore/codeserver/__init__.py index 16112eecc..9d77227f4 100644 --- a/wayflowcore/src/wayflowcore/codeserver/__init__.py +++ b/wayflowcore/src/wayflowcore/codeserver/__init__.py @@ -7,5 +7,6 @@ """Code Executor Protocol server models and services.""" from wayflowcore.codeserver.server import CodeExecutorServer +from wayflowcore.codeserver.serverstorageconfig import CodeExecutorServerStorageConfig -__all__ = ["CodeExecutorServer"] +__all__ = ["CodeExecutorServer", "CodeExecutorServerStorageConfig"] diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py index 764b27587..5e473685d 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py +++ b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py @@ -245,8 +245,10 @@ def _spawn_worker( ) -> tuple[Queue[object], Queue[object], BaseProcess]: """Create and start one local Python worker process.""" context = multiprocessing.get_context("spawn") - command_queue = cast(Queue[object], context.Queue()) - result_queue = cast(Queue[object], context.Queue()) + # multiprocessing.queues.Queue is not subscriptable at runtime on + # Python 3.11, so keep the generic form in annotations only. + command_queue = context.Queue() + result_queue = context.Queue() process = context.Process( target=worker_main, args=(command_queue, result_queue, self.policy), diff --git a/wayflowcore/src/wayflowcore/codeserver/server.py b/wayflowcore/src/wayflowcore/codeserver/server.py index 718eafe2f..e03a4df30 100644 --- a/wayflowcore/src/wayflowcore/codeserver/server.py +++ b/wayflowcore/src/wayflowcore/codeserver/server.py @@ -8,13 +8,22 @@ from __future__ import annotations -from fastapi import FastAPI +import secrets +import warnings +from ipaddress import ip_address +from typing import Any, Optional, Sequence + +from fastapi import FastAPI, Request, status +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse from wayflowcore.codeserver.app import create_code_executor_app from wayflowcore.codeserver.backend import CodeExecutorBackend from wayflowcore.codeserver.backends.local_python import LocalPythonBackend +from wayflowcore.codeserver.serverstorageconfig import CodeExecutorServerStorageConfig from wayflowcore.codeserver.service import CodeExecutionService from wayflowcore.codeserver.storage import CodeExecutorStorage +from wayflowcore.datastore import Datastore class CodeExecutorServer: @@ -23,27 +32,207 @@ class CodeExecutorServer: def __init__( self, backend: CodeExecutorBackend | None = None, - storage: CodeExecutorStorage | None = None, - *, + storage: Datastore | None = None, + storage_config: CodeExecutorServerStorageConfig | None = None, + allowed_origins: Optional[Sequence[str]] = None, + allow_credentials: bool = True, + allowed_methods: Optional[Sequence[str]] = None, + allowed_headers: Optional[Sequence[str]] = None, server_name: str = "wayflow-code-server", protocol_version: str = "26.1.3", ) -> None: - """Initialize a server with a backend and optional snapshot storage.""" + """Initialize a server with a backend and optional snapshot storage. + + Parameters + ---------- + storage: + Datastore for server persistence. When omitted, an in-memory datastore is used. + storage_config: + Configuration for the collections and serialized snapshot fields. + allowed_origins: + Origins allowed to make browser cross-origin requests to the server through + CORS (Cross-Origin Resource Sharing). CORS is a browser access-control + mechanism that decides whether JavaScript loaded from one origin, such as + ``https://app.example.com``, may call this server on another origin. + An origin is the scheme, host, and port, for example + ``https://app.example.com`` or ``http://localhost:3000``. + If not provided, CORS middleware is not enabled and browsers deny + cross-origin requests by default. + Examples: + ``["https://app.example.com"]`` allows one browser application origin. + ``["*"]`` allows any origin, but only when ``allow_credentials`` is false. + Wildcard subdomain patterns such as ``["*.example.com"]`` are not + supported here; list each allowed origin explicitly. + allow_credentials: + Whether CORS requests may include credentials. + allowed_methods: + HTTP methods accepted by CORS preflight requests. + allowed_headers: + HTTP headers accepted by CORS preflight requests. + """ self.backend = backend or LocalPythonBackend() - self.service = CodeExecutionService(backend=self.backend, storage=storage) + effective_storage_config = storage_config or CodeExecutorServerStorageConfig() + self.service = CodeExecutionService( + backend=self.backend, + storage=CodeExecutorStorage( + datastore=storage or effective_storage_config.datastore, + storage_config=effective_storage_config, + ), + ) self.server_name = server_name self.protocol_version = protocol_version + self.allowed_origins = allowed_origins + self.allow_credentials = allow_credentials + self.allowed_methods = allowed_methods + self.allowed_headers = allowed_headers + + def _setup_middleware( + self, + app: FastAPI, + allowed_origins: Optional[Sequence[str]], + allow_credentials: bool, + allowed_methods: Optional[Sequence[str]], + allowed_headers: Optional[Sequence[str]], + ) -> None: + """Set up CORS and other middleware.""" + if not allowed_origins: + # allowed_methods and allowed_headers are only meaningful once CORS is enabled + # with an origin allow-list. Without allowed_origins, keep CORS disabled so + # browser cross-origin access is denied by default. + return + if allow_credentials and "*" in allowed_origins: + raise ValueError("Wildcard CORS origins cannot be used with credentials enabled.") + app.add_middleware( + CORSMiddleware, + allow_origins=list(allowed_origins), + allow_credentials=allow_credentials, + allow_methods=list(allowed_methods or ["*"]), + allow_headers=list(allowed_headers or ["*"]), + ) def get_app(self) -> FastAPI: """Return the FastAPI application for deployment by an ASGI server.""" - return create_code_executor_app( + app = create_code_executor_app( self.service, server_name=self.server_name, protocol_version=self.protocol_version, ) + self._setup_middleware( + app, + allowed_origins=self.allowed_origins, + allow_credentials=self.allow_credentials, + allowed_methods=self.allowed_methods, + allowed_headers=self.allowed_headers, + ) + return app - def run(self, host: str = "127.0.0.1", port: int = 8765) -> None: + def run( + self, + host: str = "127.0.0.1", + port: int = 8765, + api_key: str | None = None, + ) -> None: """Run the Code Executor server with Uvicorn.""" import uvicorn - uvicorn.run(self.get_app(), host=host, port=port, reload=False) + _validate_server_auth_configuration(host=host, api_key=api_key) + if api_key is None: + warn_server_is_not_secured() + app = self.get_app() + if api_key is not None: + _add_token_authentication(app, api_key) + uvicorn.run(app, host=host, port=port, reload=False) + + +def _add_token_authentication(app: FastAPI, api_key: str) -> None: + """Add bearer-token authentication middleware to an application.""" + + @app.middleware("http") + async def require_bearer_token(request: Request, call_next: Any) -> Any: + auth_header = request.headers.get("authorization", "") + if not secrets.compare_digest(auth_header, f"Bearer {api_key}"): + return JSONResponse( + status_code=status.HTTP_401_UNAUTHORIZED, + content={"detail": "Missing or invalid bearer token"}, + ) + return await call_next(request) + + +def _validate_server_auth_configuration(host: str, api_key: str | None) -> None: + """Require authentication when binding outside the loopback interface.""" + if api_key is None and not _is_loopback_host(host): + raise ValueError( + "An api_key is required when binding to a non-loopback host. " + "Use host='127.0.0.1' for local unauthenticated development." + ) + + +def _is_loopback_host(host: str) -> bool: + normalized_host = host.strip("[]").lower() + if normalized_host == "localhost": + return True + try: + return ip_address(normalized_host).is_loopback + except ValueError: + return False + + +_WARNING_MESSAGE = r""" + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ ⚠️ SECURITY WARNING ┃ +┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +┃ ┃ +┃ This server has NO built-in authentication or encryption. ┃ +┃ Anyone with network access can invoke this CodeExecutor server. ┃ +┃ ┃ +┃ For production, either: ┃ +┃ • Deploy behind an authenticated gateway ┃ +┃ • Add auth middleware via `CodeExecutorServer.get_app()` ┃ +┃ ┃ +┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +""" + + +def warn_server_is_not_secured() -> None: + warnings.warn(_WARNING_MESSAGE) + + +_CURL_EXAMPLES = r""" + + +curl http://127.0.0.1:8010/v1/code-executor + + + +curl -X POST http://127.0.0.1:8010/v1/executions \ + -H "Content-Type: application/json" \ + -d '{ + "language_id": "python", + "input": [ + { + "type": "script", + "source_code": "print(\"hello from curl\")" + } + ], + "wait": true + }' + +curl -X POST http://127.0.0.1:8010/v1/executions \ + -H "Content-Type: application/json" \ + -d '{ + "language_id": "python", + "input": [ + { + "type": "function", + "source_code": "def multiply(a, b):\n return a * b", + "function_name": "multiply", + "arguments": { + "a": 6, + "b": 7 + } + } + ], + "wait": true + }' +""" diff --git a/wayflowcore/src/wayflowcore/codeserver/serverstorageconfig.py b/wayflowcore/src/wayflowcore/codeserver/serverstorageconfig.py new file mode 100644 index 000000000..54381ecf6 --- /dev/null +++ b/wayflowcore/src/wayflowcore/codeserver/serverstorageconfig.py @@ -0,0 +1,33 @@ +# Copyright © 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Storage configuration for the Code Executor server.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from wayflowcore.datastore import Datastore + + +@dataclass +class CodeExecutorServerStorageConfig: + """Configuration for Code Executor execution and session snapshots.""" + + datastore: Datastore | None = None + """Datastore used for persistence.""" + + executions_table_name: str = "code_executions" + """Collection containing execution snapshots.""" + + sessions_table_name: str = "code_sessions" + """Collection containing session snapshots.""" + + id_column_name: str = "id" + """Column containing the public snapshot identifier.""" + + data_column_name: str = "data" + """Column containing the serialized snapshot.""" diff --git a/wayflowcore/src/wayflowcore/codeserver/storage.py b/wayflowcore/src/wayflowcore/codeserver/storage.py index e46b19a60..bb2ba3dd3 100644 --- a/wayflowcore/src/wayflowcore/codeserver/storage.py +++ b/wayflowcore/src/wayflowcore/codeserver/storage.py @@ -12,6 +12,7 @@ from dataclasses import dataclass, field from wayflowcore.codeserver.models import ExecutionResponse, SessionSnapshot +from wayflowcore.codeserver.serverstorageconfig import CodeExecutorServerStorageConfig from wayflowcore.datastore import Datastore, Entity, InMemoryDatastore from wayflowcore.property import StringProperty @@ -21,19 +22,19 @@ _DATA_PROPERTY = "data" -def _storage_schema() -> dict[str, Entity]: +def _storage_schema(config: CodeExecutorServerStorageConfig) -> dict[str, Entity]: """Build the datastore schema used for execution and session snapshots.""" return { - _EXECUTIONS_COLLECTION: Entity( + config.executions_table_name: Entity( properties={ - _ID_PROPERTY: StringProperty(), - _DATA_PROPERTY: StringProperty(), + config.id_column_name: StringProperty(), + config.data_column_name: StringProperty(), } ), - _SESSIONS_COLLECTION: Entity( + config.sessions_table_name: Entity( properties={ - _ID_PROPERTY: StringProperty(), - _DATA_PROPERTY: StringProperty(), + config.id_column_name: StringProperty(), + config.data_column_name: StringProperty(), } ), } @@ -51,6 +52,7 @@ class CodeExecutorStorage: """ datastore: Datastore | None = None + storage_config: CodeExecutorServerStorageConfig | None = None _datastore: Datastore = field(init=False, repr=False) def __post_init__(self) -> None: @@ -58,6 +60,7 @@ def __post_init__(self) -> None: if self.datastore is not None: self._datastore = self.datastore return + config = self.storage_config or CodeExecutorServerStorageConfig() # The storage adapter explicitly documents this as its local default; # avoid repeating InMemoryDatastore's general-purpose warning here. with warnings.catch_warnings(): @@ -66,59 +69,78 @@ def __post_init__(self) -> None: message="InMemoryDatastore is for DEVELOPMENT", category=UserWarning, ) - self._datastore = InMemoryDatastore(_storage_schema()) + self._datastore = InMemoryDatastore(_storage_schema(config)) + + @property + def _config(self) -> CodeExecutorServerStorageConfig: + """Return the effective storage configuration.""" + return self.storage_config or CodeExecutorServerStorageConfig() def create_execution(self, response: ExecutionResponse) -> ExecutionResponse: """Persist and return a new execution snapshot.""" - self._create(_EXECUTIONS_COLLECTION, response.id, response.model_dump_json(by_alias=True)) + self._create( + self._config.executions_table_name, response.id, response.model_dump_json(by_alias=True) + ) return response def get_execution(self, execution_id: str) -> ExecutionResponse: """Retrieve an execution snapshot by identifier.""" - data = self._get_data(_EXECUTIONS_COLLECTION, execution_id) + data = self._get_data(self._config.executions_table_name, execution_id) return ExecutionResponse.model_validate_json(data) def update_execution(self, response: ExecutionResponse) -> ExecutionResponse: """Replace an existing execution snapshot.""" - self._update(_EXECUTIONS_COLLECTION, response.id, response.model_dump_json(by_alias=True)) + self._update( + self._config.executions_table_name, response.id, response.model_dump_json(by_alias=True) + ) return response def delete_execution(self, execution_id: str) -> None: """Delete an execution snapshot.""" - self._datastore.delete(_EXECUTIONS_COLLECTION, where={_ID_PROPERTY: execution_id}) + self._datastore.delete( + self._config.executions_table_name, + where={self._config.id_column_name: execution_id}, + ) def create_session(self, snapshot: SessionSnapshot) -> SessionSnapshot: """Persist and return a new session snapshot.""" - self._create(_SESSIONS_COLLECTION, snapshot.id, snapshot.model_dump_json(by_alias=True)) + self._create( + self._config.sessions_table_name, snapshot.id, snapshot.model_dump_json(by_alias=True) + ) return snapshot def get_session(self, session_id: str) -> SessionSnapshot: """Retrieve a session snapshot by identifier.""" - data = self._get_data(_SESSIONS_COLLECTION, session_id) + data = self._get_data(self._config.sessions_table_name, session_id) return SessionSnapshot.model_validate_json(data) def update_session(self, snapshot: SessionSnapshot) -> SessionSnapshot: """Replace an existing session snapshot.""" - self._update(_SESSIONS_COLLECTION, snapshot.id, snapshot.model_dump_json(by_alias=True)) + self._update( + self._config.sessions_table_name, snapshot.id, snapshot.model_dump_json(by_alias=True) + ) return snapshot def delete_session(self, session_id: str) -> None: """Delete a session snapshot.""" - self._datastore.delete(_SESSIONS_COLLECTION, where={_ID_PROPERTY: session_id}) + self._datastore.delete( + self._config.sessions_table_name, + where={self._config.id_column_name: session_id}, + ) def _create(self, collection_name: str, record_id: str, data: str) -> None: """Create one serialized datastore record.""" self._datastore.create( collection_name, - {_ID_PROPERTY: record_id, _DATA_PROPERTY: data}, + {self._config.id_column_name: record_id, self._config.data_column_name: data}, ) def _update(self, collection_name: str, record_id: str, data: str) -> None: """Update one serialized datastore record.""" updated = self._datastore.update( collection_name, - where={_ID_PROPERTY: record_id}, - update={_DATA_PROPERTY: data}, + where={self._config.id_column_name: record_id}, + update={self._config.data_column_name: data}, ) if not updated: raise KeyError(f"Record '{record_id}' was not found.") @@ -127,12 +149,12 @@ def _get_data(self, collection_name: str, record_id: str) -> str: """Retrieve the serialized payload for one datastore record.""" records = self._datastore.list( collection_name, - where={_ID_PROPERTY: record_id}, + where={self._config.id_column_name: record_id}, limit=1, ) if not records: raise KeyError(f"Record '{record_id}' was not found.") - data = records[0].get(_DATA_PROPERTY) + data = records[0].get(self._config.data_column_name) if not isinstance(data, str): raise TypeError(f"Record '{record_id}' contains invalid serialized data.") return data diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/_http.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/_http.py new file mode 100644 index 000000000..ed9f4d0b3 --- /dev/null +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/_http.py @@ -0,0 +1,68 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Private synchronous HTTP client for Code Executor Protocol endpoints.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from wayflowcore.codeserver.models import ( + CodeExecutionRequest, + CodeExecutorCapabilities, + ExecutionResponse, +) + + +class CodeExecutorHttpClient: + """Small HTTP client for the Code Executor Protocol routes.""" + + def __init__( + self, + base_url: str, + *, + headers: dict[str, str] | None = None, + timeout_seconds: float = 30.0, + ) -> None: + """Initialize the client with a base URL and per-request timeout.""" + self.base_url = base_url.rstrip("/") + self._client = httpx.Client( + headers=headers, + timeout=timeout_seconds, + ) + + def get_capabilities(self) -> dict[str, Any]: + """Fetch the server capabilities document.""" + response = self._client.get(f"{self.base_url}/v1/code-executor") + response.raise_for_status() + return CodeExecutorCapabilities.model_validate(response.json()).capabilities + + def create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: + """Submit an execution request.""" + response = self._client.post( + f"{self.base_url}/v1/executions", + json=request.model_dump(by_alias=True, exclude_none=True), + ) + response.raise_for_status() + return ExecutionResponse.model_validate(response.json()) + + def get_execution(self, execution_id: str) -> ExecutionResponse: + """Fetch an execution snapshot.""" + response = self._client.get(f"{self.base_url}/v1/executions/{execution_id}") + response.raise_for_status() + return ExecutionResponse.model_validate(response.json()) + + def cancel_execution(self, execution_id: str) -> ExecutionResponse: + """Cancel an execution.""" + response = self._client.post(f"{self.base_url}/v1/executions/{execution_id}/cancel") + response.raise_for_status() + return ExecutionResponse.model_validate(response.json()) + + def close(self) -> None: + """Close the underlying HTTP client.""" + self._client.close() diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/endpointexecutor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/endpointexecutor.py index 4c337dd24..0cd4f1a8b 100644 --- a/wayflowcore/src/wayflowcore/tools/codeexecutors/endpointexecutor.py +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/endpointexecutor.py @@ -5,14 +5,16 @@ # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. from dataclasses import dataclass -from typing import Dict, Optional +from typing import Any, Dict, Optional +from wayflowcore.codeserver.models import CodeExecutionRequest, ExecutionResponse from wayflowcore.retrypolicy import RetryPolicy +from ._http import CodeExecutorHttpClient from .executor import CodeExecutor -@dataclass +@dataclass(kw_only=True) class EndpointCodeExecutor(CodeExecutor): """Run code through a Code Executor endpoint.""" @@ -27,3 +29,49 @@ class EndpointCodeExecutor(CodeExecutor): retry_policy: RetryPolicy | None = None """Transport retry policy.""" + + request_timeout_seconds: float = 30.0 + """Maximum time allowed for one HTTP request.""" + + def __post_init__(self) -> None: + """Initialize the private HTTP client lazily.""" + self._client: CodeExecutorHttpClient | None = None + + def _get_http_client(self) -> CodeExecutorHttpClient: + """Return the cached private HTTP client.""" + if self._client is None: + request_headers = dict(self.headers or {}) + request_headers.update(self.sensitive_headers or {}) + timeout = ( + self.retry_policy.request_timeout + if self.retry_policy is not None + else self.request_timeout_seconds + ) + self._client = CodeExecutorHttpClient( + self.url, + headers=request_headers, + timeout_seconds=timeout, + ) + return self._client + + def _create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: + """Submit an execution through HTTP.""" + return self._get_http_client().create_execution(request) + + def _get_execution(self, execution_id: str) -> ExecutionResponse: + """Retrieve an execution snapshot through HTTP.""" + return self._get_http_client().get_execution(execution_id) + + def _cancel_execution(self, execution_id: str) -> ExecutionResponse: + """Cancel an execution through HTTP.""" + return self._get_http_client().cancel_execution(execution_id) + + def _get_capabilities(self) -> dict[str, Any]: + """Retrieve capabilities through HTTP.""" + return self._get_http_client().get_capabilities() + + def close(self) -> None: + """Close the private HTTP client.""" + if self._client is not None: + self._client.close() + self._client = None diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py index 9014c0622..44c1dd177 100644 --- a/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py @@ -1,28 +1,55 @@ -# Copyright © 2026 Oracle and/or its affiliates. +# Copyright © 2025, 2026 Oracle and/or its affiliates. # # This software is under the Apache License 2.0 # (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. -"""Base classes for code executor configurations.""" +"""Shared execution logic for Code Executor configurations.""" from __future__ import annotations +import time +from abc import ABC, abstractmethod +from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Any, Mapping, Sequence +from typing import Any -from wayflowcore.component import Component +import anyio -from ._utils import CodeExecutionStatus +from wayflowcore._utils.notgiven import NOT_GIVEN +from wayflowcore.codeserver.models import ( + TASK_STATUS_CANCELLED, + TASK_STATUS_COMPLETED, + TASK_STATUS_FAILED, + TASK_STATUS_TIMED_OUT, + CodeExecutionRequest, + ExecutionResponse, + ExecutionResult, + FunctionInput, + ScriptInput, +) +from wayflowcore.component import DataclassComponent + +from ._utils import ( + CodeExecutionCancelled, + CodeExecutionFailed, + CodeExecutionRejected, + CodeExecutionStatus, + CodeExecutionSucceeded, + CodeExecutionTimedOut, +) + +_POLL_INTERVAL_SECONDS = 0.1 @dataclass -class CodeExecutor(Component): +class CodeExecutor(DataclassComponent, ABC): """Class to configure a code executor.""" - timeout_seconds: float + timeout_seconds: float = 30.0 """Maximum wall-clock time allowed for one execution. The default value is ``30``.""" - max_code_chars: int + + max_code_chars: int = 50_000 """Maximum accepted source length in characters. The default value is ``50000``.""" def _execute_function( @@ -58,7 +85,21 @@ def _execute_function( CodeExecutionStatus A terminal execution status. """ - raise NotImplementedError + request = CodeExecutionRequest( + language_id=language, + input=[ + FunctionInput( + type="function", + source_code=self._validate_code(code), + function_name=function_name, + arguments=dict(arguments), + ) + ], + dependencies=list(dependencies), + metadata=dict(metadata or {}), + wait=False, + ) + return self._run_execution_request_to_completion(request) async def _execute_function_async( self, @@ -69,8 +110,16 @@ async def _execute_function_async( dependencies: Sequence[str] = (), metadata: Mapping[str, Any] | None = None, ) -> CodeExecutionStatus: - """Asynchronously run one named function defined in source code.""" - raise NotImplementedError + """Asynchronously run one named function and poll it.""" + return await anyio.to_thread.run_sync( + self._execute_function, + code, + language, + function_name, + arguments, + dependencies, + metadata, + ) def _execute_script( self, @@ -100,7 +149,19 @@ def _execute_script( A terminal execution status, or ``waiting_for_context`` when the server asks the host to do work. """ - raise NotImplementedError + request = CodeExecutionRequest( + language_id=language, + input=[ + ScriptInput( + type="script", + source_code=self._validate_code(code), + ) + ], + dependencies=list(dependencies), + metadata=dict(metadata or {}), + wait=False, + ) + return self._run_execution_request_to_completion(request) async def _execute_script_async( self, @@ -110,11 +171,131 @@ async def _execute_script_async( metadata: Mapping[str, Any] | None = None, ) -> CodeExecutionStatus: """Asynchronously run one script.""" - raise NotImplementedError + return await anyio.to_thread.run_sync( + self._execute_script, + code, + language, + dependencies, + metadata, + ) def get_capabilities(self) -> dict[str, Any]: - """Return capability information supplied by the configured server, - which may include the supported languages, supported execution modes, - and any other supported feature. - """ + """Return capabilities supplied by the configured execution service.""" + return self._get_capabilities() + + def _run_execution_request_to_completion( + self, request: CodeExecutionRequest + ) -> CodeExecutionStatus: + """Submit one request and poll its execution until it is terminal.""" + started_at = time.monotonic() + try: + response = self._create_execution(request) + except Exception as exc: # noqa: BLE001 - transport boundary. + return CodeExecutionRejected( + execution_id="", + message=str(exc), + ) + + while response.status not in { + TASK_STATUS_COMPLETED, + TASK_STATUS_FAILED, + TASK_STATUS_TIMED_OUT, + TASK_STATUS_CANCELLED, + }: + if time.monotonic() - started_at >= self.timeout_seconds: + try: + response = self._cancel_execution(response.id) + except Exception as exc: # noqa: BLE001 - transport boundary. + return CodeExecutionTimedOut( + execution_id=response.id, + message=str(exc), + metadata=response.metadata, + ) + return CodeExecutionTimedOut( + execution_id=response.id, + message="Execution timed out.", + metadata=response.metadata, + ) + time.sleep(_POLL_INTERVAL_SECONDS) + response = self._get_execution(response.id) + return self._response_to_status(response) + + def _validate_code(self, code: str) -> str: + """Validate the client-side source-size limit.""" + if len(code) > self.max_code_chars: + raise ValueError("Code exceeds the maximum accepted source length.") + return code + + @staticmethod + def _response_to_status(response: ExecutionResponse) -> CodeExecutionStatus: + """Convert one protocol response into a client execution status.""" + output = response.output[0] if response.output else None + metadata = dict(response.metadata) + message = metadata.get("error") # TODO: see if we need to remove this + message_text = message if isinstance(message, str) else None + + if response.status == TASK_STATUS_COMPLETED and isinstance(output, ExecutionResult): + result = output.structured_content + return CodeExecutionSucceeded( + execution_id=response.id, + stdout=_stream_text(output, "stdout"), + stderr=_stream_text(output, "stderr"), + result=result if result is not NOT_GIVEN else NOT_GIVEN, + metadata=metadata, + ) + if response.status == TASK_STATUS_FAILED: + return CodeExecutionFailed( + execution_id=response.id, + message=message_text, + stdout=_stream_text(output, "stdout"), + stderr=_stream_text(output, "stderr"), + metadata=metadata, + ) + if response.status == TASK_STATUS_TIMED_OUT: + return CodeExecutionTimedOut( + execution_id=response.id, + message=message_text, + stdout=_stream_text(output, "stdout"), + stderr=_stream_text(output, "stderr"), + metadata=metadata, + ) + if response.status == TASK_STATUS_CANCELLED: + return CodeExecutionCancelled( + execution_id=response.id, + message=message_text, + stdout=_stream_text(output, "stdout"), + stderr=_stream_text(output, "stderr"), + metadata=metadata, + ) + return CodeExecutionFailed( + execution_id=response.id, + message=f"Unsupported execution response status: {response.status}", + metadata=metadata, + ) + + @abstractmethod + def _create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: + """Submit an execution request through the configured transport.""" raise NotImplementedError + + @abstractmethod + def _get_execution(self, execution_id: str) -> ExecutionResponse: + """Retrieve an execution snapshot through the configured transport.""" + raise NotImplementedError + + @abstractmethod + def _cancel_execution(self, execution_id: str) -> ExecutionResponse: + """Cancel an execution through the configured transport.""" + raise NotImplementedError + + @abstractmethod + def _get_capabilities(self) -> dict[str, Any]: + """Retrieve capabilities through the configured transport.""" + raise NotImplementedError + + +def _stream_text(output: object, stream: str) -> str: + """Extract captured text for one stream from an execution result.""" + if not isinstance(output, ExecutionResult): + return "" + return "".join(block.text for block in output.content if block.stream == stream) diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/localcontainerexecutor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/localcontainerexecutor.py index 7e5c3a031..b98f43f29 100644 --- a/wayflowcore/src/wayflowcore/tools/codeexecutors/localcontainerexecutor.py +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/localcontainerexecutor.py @@ -10,7 +10,7 @@ from .executor import CodeExecutor -@dataclass +@dataclass(kw_only=True) class LocalContainerCodeExecutor(CodeExecutor): """Run code through a locally started container-backed Code Executor.""" diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py index 033ad2f23..0ad50241c 100644 --- a/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py @@ -5,24 +5,174 @@ # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. +import json from contextlib import contextmanager -from dataclasses import dataclass -from typing import Iterator +from contextvars import ContextVar +from dataclasses import dataclass, field +from multiprocessing import Process, Queue +from multiprocessing.queues import Queue as QueueType +from queue import Empty +from typing import Any, Iterator + +from wayflowcore.codeserver import CodeExecutorServer +from wayflowcore.codeserver.models import ( + CodeExecutionRequest, + CodeExecutorCapabilities, + ExecutionResponse, +) from .executor import CodeExecutor +_SUBPROCESS_EXECUTION_ENABLED: ContextVar[bool] = ContextVar( + "_SUBPROCESS_EXECUTION_ENABLED", default=False +) +_DEFAULT_REQUEST_TIMEOUT_SECONDS = 30.0 +_CODE_EXECUTOR_PROTOCOL_VERSION = "26.1.3" +_CODE_EXECUTOR_SERVER_NAME = "wayflow-code-server" +_MAX_LIVE_PROCESSES = 4 +_MAX_QUEUED_REQUESTS = 32 + + +def _subprocess_worker(request_queue: QueueType[str], response_queue: QueueType[str]) -> None: + """Serve serialized Code Executor requests in a child process.""" + server = CodeExecutorServer() + service = server.service + while True: + message = request_queue.get() + if message == "shutdown": + return + try: + command = _decode_command(message) + operation = command["operation"] + if operation == "capabilities": + result: Any = CodeExecutorCapabilities( + protocol_version=_CODE_EXECUTOR_PROTOCOL_VERSION, + server_name=_CODE_EXECUTOR_SERVER_NAME, + capabilities=service.backend.get_capabilities(), + ).model_dump_json(by_alias=True) + elif operation == "create_execution": + request = CodeExecutionRequest.model_validate_json(command["payload"]) + result = service.execute(request).model_dump_json(by_alias=True) + elif operation == "get_execution": + result = service.get_execution(command["execution_id"]).model_dump_json( + by_alias=True + ) + elif operation == "cancel_execution": + result = service.cancel_execution(command["execution_id"]).model_dump_json( + by_alias=True + ) + else: + raise ValueError(f"Unsupported subprocess operation: {operation}") + response_queue.put(_encode_response({"result": result})) + except Exception as exc: # noqa: BLE001 - child process boundary. + response_queue.put(_encode_response({"error": str(exc)})) + + +def _decode_command(message: str) -> dict[str, str]: + """Decode one parent-to-worker command.""" + value = json.loads(message) + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise ValueError("Subprocess command must be a JSON object.") + return value -@dataclass + +def _encode_response(value: object) -> str: + """Encode one worker response.""" + return json.dumps(value) + + +@dataclass(kw_only=True) class SubProcessCodeExecutor(CodeExecutor): """Run code through a Code Executor subprocess.""" + request_timeout_seconds: float = _DEFAULT_REQUEST_TIMEOUT_SECONDS + """Maximum time allowed for one IPC request.""" + + _request_queue: QueueType[str] | None = field(default=None, init=False, repr=False) + _response_queue: QueueType[str] | None = field(default=None, init=False, repr=False) + _process: Process | None = field(default=None, init=False, repr=False) + + def _ensure_worker(self) -> None: + """Start the child server process on first use.""" + if not _SUBPROCESS_EXECUTION_ENABLED.get(): + raise RuntimeError("Subprocess code execution requires subprocess_execution_enabled().") + if self._process is not None and self._process.is_alive(): + return + self._request_queue = Queue() + self._response_queue = Queue() + self._process = Process( + target=_subprocess_worker, + args=(self._request_queue, self._response_queue), + ) + self._process.start() + + def _request(self, command: dict[str, str]) -> dict[str, Any]: + """Send one command to the child server and await its response.""" + self._ensure_worker() + if self._request_queue is None or self._response_queue is None: + raise RuntimeError("Subprocess worker queues are unavailable.") + self._request_queue.put(_encode_response(command)) + try: + response = self._response_queue.get(timeout=self.request_timeout_seconds) + except Empty as exc: + raise TimeoutError("Subprocess request timed out.") from exc + value = json.loads(response) + if not isinstance(value, dict): + raise RuntimeError("Subprocess response must be a JSON object.") + if "error" in value: + raise RuntimeError(str(value["error"])) + return value + + def _create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: + """Submit an execution request through the subprocess worker.""" + value = self._request( + { + "operation": "create_execution", + "payload": request.model_dump_json(by_alias=True), + } + ) + return ExecutionResponse.model_validate_json(str(value["result"])) + + def _get_execution(self, execution_id: str) -> ExecutionResponse: + """Retrieve an execution snapshot through the subprocess worker.""" + value = self._request({"operation": "get_execution", "execution_id": execution_id}) + return ExecutionResponse.model_validate_json(str(value["result"])) + + def _cancel_execution(self, execution_id: str) -> ExecutionResponse: + """Cancel an execution through the subprocess worker.""" + value = self._request({"operation": "cancel_execution", "execution_id": execution_id}) + return ExecutionResponse.model_validate_json(str(value["result"])) + + def _get_capabilities(self) -> dict[str, Any]: + """Retrieve capabilities through the subprocess worker.""" + value = self._request({"operation": "capabilities"}) + return CodeExecutorCapabilities.model_validate_json(str(value["result"])).capabilities + + def close(self) -> None: + """Stop the child server process and release its queues.""" + if self._process is None: + return + if self._process.is_alive() and self._request_queue is not None: + self._request_queue.put("shutdown") + self._process.join(timeout=self.request_timeout_seconds) + if self._process.is_alive(): + self._process.terminate() + self._process.join() + self._request_queue = None + self._response_queue = None + self._process = None + @contextmanager def subprocess_execution_enabled() -> Iterator[None]: """ Temporarily enable subprocess code execution in the current context. """ - yield + token = _SUBPROCESS_EXECUTION_ENABLED.set(True) + try: + yield + finally: + _SUBPROCESS_EXECUTION_ENABLED.reset(token) def configure_subprocess_executor_runtime( @@ -39,4 +189,10 @@ def configure_subprocess_executor_runtime( max_queued_requests Maximum number of execution requests that may wait for worker capacity. """ - raise NotImplementedError + global _MAX_LIVE_PROCESSES, _MAX_QUEUED_REQUESTS + if max_live_processes <= 0: + raise ValueError("max_live_processes must be positive.") + if max_queued_requests < 0: + raise ValueError("max_queued_requests must be non-negative.") + _MAX_LIVE_PROCESSES = max_live_processes + _MAX_QUEUED_REQUESTS = max_queued_requests diff --git a/wayflowcore/tests/tools/codeexecutors/conftest.py b/wayflowcore/tests/tools/codeexecutors/conftest.py new file mode 100644 index 000000000..8f53c2a8a --- /dev/null +++ b/wayflowcore/tests/tools/codeexecutors/conftest.py @@ -0,0 +1,56 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Fixtures for CodeExecutor integration tests.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from tests.utils import _check_server_is_up, _terminate_process_tree, get_available_port + +WAYFLOW_CODE_EXECUTOR_CONTAINER_IMAGE = os.environ.get("WAYFLOW_CODE_EXECUTOR_CONTAINER_IMAGE") + + +@pytest.fixture(scope="session") +def code_executor_url(session_tmp_path: Path) -> str: + """Lazily start one local Code Executor endpoint for HTTP tests.""" + port = get_available_port(session_tmp_path) + process = subprocess.Popen( + [ + sys.executable, + "-u", + str(Path(__file__).with_name("start_codeserver.py")), + "--host", + "127.0.0.1", + "--port", + str(port), + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + url = f"http://127.0.0.1:{port}" + try: + deadline = time.monotonic() + 10.0 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("Code Executor server exited before becoming ready.") + if _check_server_is_up(f"{url}/v1/code-executor", timeout_s=0.5): + break + time.sleep(0.2) + else: + raise RuntimeError("Code Executor server did not become ready in time.") + yield url + finally: + _terminate_process_tree(process) diff --git a/wayflowcore/tests/tools/codeexecutors/start_codeserver.py b/wayflowcore/tests/tools/codeexecutors/start_codeserver.py new file mode 100644 index 000000000..5576c55b0 --- /dev/null +++ b/wayflowcore/tests/tools/codeexecutors/start_codeserver.py @@ -0,0 +1,26 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""Start a local Code Executor server for executor integration tests.""" + +from __future__ import annotations + +import argparse + +from wayflowcore.codeserver import CodeExecutorServer + + +def main() -> None: + """Start the local Python Code Executor server.""" + parser = argparse.ArgumentParser() + parser.add_argument("--host", required=True) + parser.add_argument("--port", required=True, type=int) + args = parser.parse_args() + CodeExecutorServer().run(host=args.host, port=args.port) + + +if __name__ == "__main__": + main() diff --git a/wayflowcore/tests/tools/codeexecutors/test_codeexecutors_e2e.py b/wayflowcore/tests/tools/codeexecutors/test_codeexecutors_e2e.py new file mode 100644 index 000000000..84f5489b6 --- /dev/null +++ b/wayflowcore/tests/tools/codeexecutors/test_codeexecutors_e2e.py @@ -0,0 +1,152 @@ +# Copyright © 2025, 2026 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""End-to-end tests shared by the available CodeExecutor implementations.""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any + +import pytest + +from tests.tools.codeexecutors.conftest import WAYFLOW_CODE_EXECUTOR_CONTAINER_IMAGE +from wayflowcore.tools.codeexecutors import ( + CodeExecutor, + EndpointCodeExecutor, + LocalContainerCodeExecutor, + SubProcessCodeExecutor, + subprocess_execution_enabled, +) +from wayflowcore.tools.codeexecutors._utils import ( + CodeExecutionFailed, + CodeExecutionSucceeded, + CodeExecutionTimedOut, +) + + +@dataclass(frozen=True) +class ExecutorConfig: + """Configuration used to instantiate one E2E executor.""" + + name: str + executor_type: type[CodeExecutor] + kwargs: dict[str, Any] + + +ALL_AVAILABLE_EXECUTORS = [ + ExecutorConfig( + name="subprocess", + executor_type=SubProcessCodeExecutor, + kwargs={"timeout_seconds": 2.0, "max_code_chars": 50_000}, + ), + ExecutorConfig( + name="endpoint", + executor_type=EndpointCodeExecutor, + kwargs={"timeout_seconds": 2.0, "max_code_chars": 50_000}, + ), +] + +if WAYFLOW_CODE_EXECUTOR_CONTAINER_IMAGE: + ALL_AVAILABLE_EXECUTORS.append( + ExecutorConfig( + name="local-container", + executor_type=LocalContainerCodeExecutor, + kwargs={ + "image": WAYFLOW_CODE_EXECUTOR_CONTAINER_IMAGE, + "timeout_seconds": 2.0, + "max_code_chars": 50_000, + }, + ) + ) + + +with_all_code_executors = pytest.mark.parametrize( + "executor_config", + argvalues=ALL_AVAILABLE_EXECUTORS, + ids=[config.name for config in ALL_AVAILABLE_EXECUTORS], +) + + +@pytest.fixture +def code_executor( + executor_config: ExecutorConfig, + request: pytest.FixtureRequest, +) -> Iterator[CodeExecutor]: + """Yield one available executor and clean up its runtime afterward.""" + kwargs = dict(executor_config.kwargs) + if executor_config.executor_type is EndpointCodeExecutor: + kwargs["url"] = request.getfixturevalue("code_executor_url") + executor = executor_config.executor_type(**kwargs) + try: + if isinstance(executor, SubProcessCodeExecutor): + with subprocess_execution_enabled(): + yield executor + else: + yield executor + finally: + close = getattr(executor, "close", None) + if callable(close): + close() + + +@with_all_code_executors +def test_code_executor_e2e_runs_script_and_captures_stdout( + code_executor: CodeExecutor, +) -> None: + """Runs a script and returns captured standard output.""" + status = code_executor._execute_script("print('hello')", "python") + + assert isinstance(status, CodeExecutionSucceeded) + assert status.stdout == "hello\n" + + +@with_all_code_executors +def test_code_executor_e2e_runs_function_with_structured_result( + code_executor: CodeExecutor, +) -> None: + """Runs a function and returns its structured result.""" + status = code_executor._execute_function( + "def multiply(a, b):\n return a * b", + "python", + "multiply", + {"a": 6, "b": 7}, + ) + + assert isinstance(status, CodeExecutionSucceeded) + assert status.result == 42 + + +@with_all_code_executors +def test_code_executor_e2e_captures_stderr(code_executor: CodeExecutor) -> None: + """Captures standard error output from a script.""" + status = code_executor._execute_script( + "import sys\nprint('warning', file=sys.stderr)", + "python", + ) + + assert isinstance(status, CodeExecutionSucceeded) + assert status.stderr == "warning\n" + + +@with_all_code_executors +def test_code_executor_e2e_returns_failed_status(code_executor: CodeExecutor) -> None: + """Returns a failed status when user code raises an exception.""" + status = code_executor._execute_script("raise ValueError('boom')", "python") + + assert isinstance(status, CodeExecutionFailed) + assert status.message is not None + + +@with_all_code_executors +def test_code_executor_e2e_returns_timed_out_status(code_executor: CodeExecutor) -> None: + """Returns a timed-out status when execution exceeds its deadline.""" + code_executor.timeout_seconds = 0.1 + + status = code_executor._execute_script("while True: pass", "python") + + assert isinstance(status, CodeExecutionTimedOut) From 0f81effca7977d7f10f0946706066776896bdf10 Mon Sep 17 00:00:00 2001 From: Paul Cayet Date: Mon, 20 Jul 2026 13:31:07 +0000 Subject: [PATCH 6/6] fix issues --- wayflowcore/src/wayflowcore/cli/codeserver.py | 54 +++++++++++++++---- .../codeserver/backends/local_python.py | 6 +++ wayflowcore/src/wayflowcore/exceptions.py | 10 ++++ .../wayflowcore/tools/codeexecutors/_http.py | 23 +++++--- .../tools/codeexecutors/executor.py | 3 +- .../tools/codeexecutors/subprocessexecutor.py | 30 +++++++---- 6 files changed, 101 insertions(+), 25 deletions(-) diff --git a/wayflowcore/src/wayflowcore/cli/codeserver.py b/wayflowcore/src/wayflowcore/cli/codeserver.py index 183a031b7..add6f30b7 100644 --- a/wayflowcore/src/wayflowcore/cli/codeserver.py +++ b/wayflowcore/src/wayflowcore/cli/codeserver.py @@ -12,6 +12,11 @@ import os from wayflowcore.codeserver import CodeExecutorServer +from wayflowcore.codeserver.backend import CodeExecutorBackend +from wayflowcore.codeserver.backends.local_python import LocalPythonBackend +from wayflowcore.codeserver.backends.pythonexecutionpolicy import ( + StrictPythonExecutionPolicy, +) __all__ = ["add_parser", "codeserver"] @@ -36,6 +41,18 @@ def add_parser( type=int, help="Port to bind (default: 8765).", ) + parser.add_argument( + "--backend", + choices=("python",), + default="python", + help="Backend to use (default: python).", + ) + parser.add_argument( + "--policy", + choices=("strict", "none"), + default="strict", + help="Python execution policy (default: strict).", + ) parser.add_argument( "--api-key", default=os.getenv("WAYFLOW_API_KEY"), @@ -47,16 +64,35 @@ def add_parser( def _run_codeserver(args: argparse.Namespace) -> None: """Run the configured local Python Code Executor server.""" - codeserver(host=args.host, port=args.port, api_key=args.api_key) + codeserver( + host=args.host, + port=args.port, + api_key=args.api_key, + backend=args.backend, + policy=args.policy, + ) -def codeserver(host: str = "127.0.0.1", port: int = 8765, api_key: str | None = None) -> None: - """Run a local Python Code Executor server.""" - CodeExecutorServer().run(host=host, port=port, api_key=api_key) +def _create_backend(backend_name: str, policy_name: str) -> CodeExecutorBackend: + """Create the configured code execution backend.""" + if backend_name != "python": + raise ValueError(f"Unsupported backend: {backend_name}") + policy = StrictPythonExecutionPolicy() if policy_name == "strict" else None + return LocalPythonBackend(policy=policy) -# Copyright © 2025, 2026 Oracle and/or its affiliates. -# -# This software is under the Apache License 2.0 -# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License -# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +def codeserver( + host: str = "127.0.0.1", + port: int = 8765, + api_key: str | None = None, + backend: str = "python", + policy: str = "strict", +) -> None: + """Run a local Python Code Executor server.""" + configured_backend = _create_backend(backend, policy) + CodeExecutorServer(backend=configured_backend).run( + host=host, + port=port, + api_key=api_key, + ) diff --git a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py index 5e473685d..5ef3516b2 100644 --- a/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py +++ b/wayflowcore/src/wayflowcore/codeserver/backends/local_python.py @@ -59,6 +59,12 @@ class LocalPythonBackend(CodeExecutorBackend): max_stderr_chars: int = 50_000 """Maximum captured standard-error characters per execution.""" + # TODO: Move live-session registry ownership to CodeExecutionService (or a dedicated + # runtime SessionRegistry). The backend should create and operate on BackendSession + # handles supplied by the service, while CodeExecutorStorage should remain responsible + # only for persisted public session snapshots. This will also keep backend configuration + # independent from live process and queue state, which is important when passing a backend + # configuration into SubProcessCodeExecutor. _sessions: dict[str, LocalPythonSession] = field(default_factory=dict, init=False, repr=False) def get_capabilities(self) -> dict[str, JsonValue]: diff --git a/wayflowcore/src/wayflowcore/exceptions.py b/wayflowcore/src/wayflowcore/exceptions.py index 3cf607d1c..3e239a521 100644 --- a/wayflowcore/src/wayflowcore/exceptions.py +++ b/wayflowcore/src/wayflowcore/exceptions.py @@ -102,3 +102,13 @@ def __init__(self, status: "AuthChallengeRequestStatus") -> None: def __str__(self) -> str: return "AuthInterrupt: Requesting auth challenge to be completed." + + +class CodeServerError(WayFlowException): + """Raised when a request to a code server fails.""" + + def __init__(self, detail: str): + self.detail = detail + + def __str__(self) -> str: + return f"Request failed with: {self.detail}" diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/_http.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/_http.py index ed9f4d0b3..bb2854460 100644 --- a/wayflowcore/src/wayflowcore/tools/codeexecutors/_http.py +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/_http.py @@ -17,6 +17,7 @@ CodeExecutorCapabilities, ExecutionResponse, ) +from wayflowcore.exceptions import CodeServerError class CodeExecutorHttpClient: @@ -48,21 +49,31 @@ def create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: f"{self.base_url}/v1/executions", json=request.model_dump(by_alias=True, exclude_none=True), ) - response.raise_for_status() - return ExecutionResponse.model_validate(response.json()) + return _convert_response(response) def get_execution(self, execution_id: str) -> ExecutionResponse: """Fetch an execution snapshot.""" response = self._client.get(f"{self.base_url}/v1/executions/{execution_id}") - response.raise_for_status() - return ExecutionResponse.model_validate(response.json()) + return _convert_response(response) def cancel_execution(self, execution_id: str) -> ExecutionResponse: """Cancel an execution.""" response = self._client.post(f"{self.base_url}/v1/executions/{execution_id}/cancel") - response.raise_for_status() - return ExecutionResponse.model_validate(response.json()) + return _convert_response(response) def close(self) -> None: """Close the underlying HTTP client.""" self._client.close() + + +def _convert_response(response: httpx.Response) -> ExecutionResponse: + try: + response.raise_for_status() + return ExecutionResponse.model_validate(response.json()) + except httpx.HTTPStatusError as e: + try: + detail = e.response.json().get("detail", "None") + except ValueError: + detail = e.response.text or "None" + + raise CodeServerError(detail=detail) from e diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py index 44c1dd177..c6354a12e 100644 --- a/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/executor.py @@ -29,6 +29,7 @@ ScriptInput, ) from wayflowcore.component import DataclassComponent +from wayflowcore.exceptions import CodeServerError from ._utils import ( CodeExecutionCancelled, @@ -190,7 +191,7 @@ def _run_execution_request_to_completion( started_at = time.monotonic() try: response = self._create_execution(request) - except Exception as exc: # noqa: BLE001 - transport boundary. + except CodeServerError as exc: # noqa: BLE001 - transport boundary. return CodeExecutionRejected( execution_id="", message=str(exc), diff --git a/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py b/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py index 0ad50241c..357daf6a5 100644 --- a/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py +++ b/wayflowcore/src/wayflowcore/tools/codeexecutors/subprocessexecutor.py @@ -15,11 +15,14 @@ from typing import Any, Iterator from wayflowcore.codeserver import CodeExecutorServer +from wayflowcore.codeserver.backend import CodeExecutorBackend +from wayflowcore.codeserver.backends.local_python import LocalPythonBackend from wayflowcore.codeserver.models import ( CodeExecutionRequest, CodeExecutorCapabilities, ExecutionResponse, ) +from wayflowcore.exceptions import CodeServerError from .executor import CodeExecutor @@ -33,9 +36,13 @@ _MAX_QUEUED_REQUESTS = 32 -def _subprocess_worker(request_queue: QueueType[str], response_queue: QueueType[str]) -> None: +def _subprocess_worker( + request_queue: QueueType[str], + response_queue: QueueType[str], + backend: CodeExecutorBackend, +) -> None: """Serve serialized Code Executor requests in a child process.""" - server = CodeExecutorServer() + server = CodeExecutorServer(backend=backend) service = server.service while True: message = request_queue.get() @@ -85,6 +92,8 @@ def _encode_response(value: object) -> str: class SubProcessCodeExecutor(CodeExecutor): """Run code through a Code Executor subprocess.""" + backend: CodeExecutorBackend = field(default_factory=LocalPythonBackend) + request_timeout_seconds: float = _DEFAULT_REQUEST_TIMEOUT_SECONDS """Maximum time allowed for one IPC request.""" @@ -102,7 +111,7 @@ def _ensure_worker(self) -> None: self._response_queue = Queue() self._process = Process( target=_subprocess_worker, - args=(self._request_queue, self._response_queue), + args=(self._request_queue, self._response_queue, self.backend), ) self._process.start() @@ -125,12 +134,15 @@ def _request(self, command: dict[str, str]) -> dict[str, Any]: def _create_execution(self, request: CodeExecutionRequest) -> ExecutionResponse: """Submit an execution request through the subprocess worker.""" - value = self._request( - { - "operation": "create_execution", - "payload": request.model_dump_json(by_alias=True), - } - ) + try: + value = self._request( + { + "operation": "create_execution", + "payload": request.model_dump_json(by_alias=True), + } + ) + except RuntimeError as e: + raise CodeServerError(detail=str(e)) return ExecutionResponse.model_validate_json(str(value["result"])) def _get_execution(self, execution_id: str) -> ExecutionResponse: