diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2bd66b3..7d8291d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,10 +1,7 @@ -## What does this PR change? -- - -## Which module? -- +## Summary +Explain what changed and why. ## Checklist -- [ ] README updated (how to run + demo) -- [ ] Inputs/outputs documented -- [ ] No credentials / private data +- [ ] No secrets/credentials committed +- [ ] Docs updated (if needed) +- [ ] Tests added/updated (if applicable) diff --git a/.gitignore b/.gitignore index 2fb32df..a7498e5 100644 --- a/.gitignore +++ b/.gitignore @@ -144,6 +144,7 @@ ENV/ env.bak/ venv.bak/ + # Spyder project settings .spyderproject .spyproject @@ -236,5 +237,17 @@ __pycache__/ .DS_Store Thumbs.db +# Monitor module generated outputs +modules/monitor/data/ +modules/monitor/deliverables/ +modules/monitor/embedding/ +modules/monitor/*.log +modules/monitor/*.dat +modules/monitor/**/*.db +modules/monitor/**/*.db-journal + OpenPolicyStack/ +# Allow example env templates (must be AFTER any .env / .env.* ignores) +!.env.example +!**/.env.example \ No newline at end of file diff --git a/ADRIAN_CON_THESIS_SCOPE.md b/ADRIAN_CON_THESIS_SCOPE.md new file mode 100644 index 0000000..9d57b53 --- /dev/null +++ b/ADRIAN_CON_THESIS_SCOPE.md @@ -0,0 +1,46 @@ +# Thesis Scope – Adrian Con García + +This repository is part of the broader OpenPolicyStack project. +This document clarifies the scope of the work developed and evaluated in the corresponding thesis. + +## Scope of Contribution + +The thesis focuses specifically on the design, implementation, and evaluation of the **orchestration layer** and its integration interface. + +The evaluated software artifact consists of: + +- `modules/orchestrator/` → central orchestration service (primary contribution) +- `modules/integration-pilot/` → controlled validation module used to test integration and evaluation conditions +- `compose.yaml` → minimal deployment configuration used to run the system + +Other modules present in the repository are part of the wider collaborative project and are **not part of the evaluated contribution**. + +## Evaluated System State + +The version contained in this branch corresponds to the **instrumented evaluation state** of the system. + +Starting from a working end-to-end orchestration prototype (baseline MVP), the system was incrementally extended to enable empirical evaluation of the following properties: + +- reproducibility +- traceability +- artifact integrity +- execution trace reconstruction +- failure handling robustness + +These properties were evaluated through a structured experimental framework (E1–E5) as described in the thesis. + +## Important Notes + +- The orchestrator was extended with structured metadata capture and hashing mechanisms to support empirical validation. +- The integration-pilot module was intentionally used as a controlled environment to isolate and test orchestration behavior before integrating external modules. +- A controlled failure trigger used exclusively for evaluation purposes has been disabled in this version. + +## How to Navigate + +For reviewers interested in the evaluated artifact: + +1. Start with: `modules/orchestrator/` +2. See integration behavior in: `modules/integration-pilot/` +3. Use `compose.yaml` to understand how services are connected + +This subset of the repository corresponds to the system evaluated in the thesis. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..8040aa2 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,81 @@ +name: openpolicystack-pilot + +services: + orchestrator: + build: + context: ./modules/orchestrator + dockerfile: Dockerfile + image: openpolicystack/orchestrator:dev + env_file: + - ./modules/orchestrator/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: orchestrator + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + ORCHESTRATOR__SQLITE_PATH: /var/openpolicystack/metadata/orchestrator.db + ORCHESTRATOR__INTEGRATION_PILOT_URL: http://integration-pilot:8080 + ports: + - "8100:8080" + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + - ops-metadata:/var/openpolicystack/metadata + networks: + - ops-core + depends_on: + - integration-pilot + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')" + ] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + integration-pilot: + build: + context: ./modules/integration-pilot + dockerfile: Dockerfile + image: openpolicystack/integration-pilot:dev + env_file: + - ./modules/integration-pilot/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: integration-pilot + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + OPS_ORCHESTRATOR_URL: http://orchestrator:8080 + PILOT_MODULE_VERSION: 0.1.1 + ports: + - "8101:8080" + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + networks: + - ops-core + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')" + ] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + +networks: + ops-core: + driver: bridge + +volumes: + ops-artifacts: + ops-metadata: \ No newline at end of file diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..0f272f9 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,108 @@ +# OpenPolicyStack – Deployment & Validation Guide + +## Purpose +This folder contains deployment-related artifacts for OpenPolicyStack, including the full architecture skeleton and basic validation instructions. + +At this stage, the system is validated through a **minimal runnable pilot stack** composed of: +- `orchestrator` (system coordinator) +- `integration-pilot` (reference module) + +--- + +## Deployment Files + +- `../compose.yaml` + → Current **runnable pilot stack** (validated baseline) + +- `compose.target-skeleton.yaml` + → **Full intended architecture** (not yet fully runnable; used as integration target) + +--- + +## Pilot Validation Procedure + +Run all commands from the repository root: + +```bash +docker compose config +docker compose build +docker compose up -d +docker compose ps +``` +Test the orchestrator: + +``` +curl http://localhost:8100/health +``` + +Execute a sample workflow: + +``` +curl-X POST http://localhost:8100/execute \ +-H"Content-Type: application/json" \ +-d'{"test":"hello","source":"vm-check"}' +``` + +Inspect shared artifacts: + +``` +docker compose exec orchestratorls-R /var/openpolicystack/artifacts +``` + +Inspect metadata: + +``` +docker compose exec orchestratorls-R /var/openpolicystack/metadata +``` + +--- + +## Expected Outcome + +A successful validation should confirm: + +- Both containers build and run successfully +- Both services report **healthy** status +- Orchestrator responds on `http://localhost:8100` +- Orchestrator can resolve `integration-pilot` via Docker network +- Shared artifact volume is written and visible across containers +- Metadata database (`orchestrator.db`) is created + +--- + +## Current Scope + +The pilot validates: + +- Basic orchestration flow (`/execute`) +- Service-to-service communication +- Shared artifact storage +- Metadata persistence (SQLite) +- Contract-compliant module execution + +Not yet covered: + +- Multi-module integration +- Real module onboarding from teammates +- Full evaluation framework (E1–E5) +- Production deployment considerations + +--- + +## Common Issues + +- Missing `.env` files (must be created from `.env.example`) +- Incorrect file paths or build contexts +- Missing or incorrect Dockerfile +- Wrong application entrypoint (`app.main`) +- Running commands outside repo root + +--- + +## Next Steps + +- Expand orchestrator metadata layer (`runs`, `module_calls`, `artifacts`) +- Strengthen determinism and reproducibility guarantees +- Introduce evaluation tests (E1–E5) +- Onboard first real module into the stack +- Progressively align with `compose.target-skeleton.yaml` \ No newline at end of file diff --git a/deploy/compose.target-skeleton.yaml b/deploy/compose.target-skeleton.yaml new file mode 100644 index 0000000..fbfb95d --- /dev/null +++ b/deploy/compose.target-skeleton.yaml @@ -0,0 +1,194 @@ +name: openpolicystack-mvp + +services: + orchestrator: + build: + context: ./orchestrator + dockerfile: Dockerfile + image: openpolicystack/orchestrator:dev + env_file: + - ./orchestrator/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: orchestrator + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + ORCHESTRATOR__SQLITE_PATH: /var/openpolicystack/metadata/orchestrator.db + ORCHESTRATOR__DATA_LAYER_URL: http://data-layer:8080 + ORCHESTRATOR__DR_ANTICORRUPTION_URL: http://dr-anticorruption:8080 + ORCHESTRATOR__MONITOR_URL: http://monitor:8080 + ORCHESTRATOR__POLICY_SIMULATOR_URL: http://policy-simulator:8080 + ORCHESTRATOR__STRATEGY_AGENT_URL: http://strategy-agent:8080 + ORCHESTRATOR__SUPPLYCHAIN_RISK_URL: http://supplychain-risk:8080 + ports: + - "8100:8080" + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + - ops-metadata:/var/openpolicystack/metadata + networks: + - ops-core + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + data-layer: + build: + context: ./modules/data-layer + dockerfile: Dockerfile + image: openpolicystack/data-layer:dev + env_file: + - ./modules/data-layer/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: data-layer + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + OPS_ORCHESTRATOR_URL: http://orchestrator:8080 + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + networks: + - ops-core + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + dr-anticorruption: + build: + context: ./modules/dr-anticorruption + dockerfile: Dockerfile + image: openpolicystack/dr-anticorruption:dev + env_file: + - ./modules/dr-anticorruption/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: dr-anticorruption + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + OPS_ORCHESTRATOR_URL: http://orchestrator:8080 + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + networks: + - ops-core + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + monitor: + build: + context: ./modules/monitor + dockerfile: Dockerfile + image: openpolicystack/monitor:dev + env_file: + - ./modules/monitor/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: monitor + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + OPS_ORCHESTRATOR_URL: http://orchestrator:8080 + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + networks: + - ops-core + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + policy-simulator: + build: + context: ./modules/policy-simulator + dockerfile: Dockerfile + image: openpolicystack/policy-simulator:dev + env_file: + - ./modules/policy-simulator/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: policy-simulator + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + OPS_ORCHESTRATOR_URL: http://orchestrator:8080 + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + networks: + - ops-core + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + strategy-agent: + build: + context: ./modules/strategy-agent + dockerfile: Dockerfile + image: openpolicystack/strategy-agent:dev + env_file: + - ./modules/strategy-agent/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: strategy-agent + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + OPS_ORCHESTRATOR_URL: http://orchestrator:8080 + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + networks: + - ops-core + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + + supplychain-risk: + build: + context: ./modules/supplychain-risk + dockerfile: Dockerfile + image: openpolicystack/supplychain-risk:dev + env_file: + - ./modules/supplychain-risk/.env + environment: + OPS_ENV: dev + OPS_MODULE_NAME: supplychain-risk + OPS_PORT: 8080 + OPS_LOG_LEVEL: INFO + OPS_ARTIFACT_ROOT: /var/openpolicystack/artifacts + OPS_ORCHESTRATOR_URL: http://orchestrator:8080 + volumes: + - ops-artifacts:/var/openpolicystack/artifacts + networks: + - ops-core + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8080/health || exit 1"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + +networks: + ops-core: + driver: bridge + +volumes: + ops-artifacts: + ops-metadata: \ No newline at end of file diff --git a/modules/integration-pilot/.env.example b/modules/integration-pilot/.env.example new file mode 100644 index 0000000..d5f5f8f --- /dev/null +++ b/modules/integration-pilot/.env.example @@ -0,0 +1,3 @@ +OPS_ENV=dev +OPS_LOG_LEVEL=INFO +PILOT_MODULE_VERSION=0.1.0 \ No newline at end of file diff --git a/modules/integration-pilot/Dockerfile b/modules/integration-pilot/Dockerfile new file mode 100644 index 0000000..7b7d624 --- /dev/null +++ b/modules/integration-pilot/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app + +EXPOSE 8080 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/modules/integration-pilot/app/__init__.py b/modules/integration-pilot/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/integration-pilot/app/main.py b/modules/integration-pilot/app/main.py new file mode 100644 index 0000000..09c6d63 --- /dev/null +++ b/modules/integration-pilot/app/main.py @@ -0,0 +1,91 @@ +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Dict + +from fastapi import FastAPI + +app = FastAPI(title="OpenPolicyStack Integration Pilot", version="0.1.0") + +MODULE_NAME = os.getenv("OPS_MODULE_NAME", "integration-pilot") +MODULE_VERSION = os.getenv("PILOT_MODULE_VERSION", "0.1.1") +ARTIFACT_ROOT = Path(os.getenv("OPS_ARTIFACT_ROOT", "/var/openpolicystack/artifacts")) + + +def canonical_json(value: Any) -> str: + """ + Deterministic JSON serialization used for stable content hashing. + """ + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +@app.get("/health") +def health() -> Dict[str, Any]: + return { + "status": "ok", + "module_name": MODULE_NAME, + "version": MODULE_VERSION, + } + + +@app.post("/execute") +def execute(payload: Dict[str, Any]) -> Dict[str, Any]: + module_dir = ARTIFACT_ROOT / MODULE_NAME + module_dir.mkdir(parents=True, exist_ok=True) + + stable_input = payload.get("input", payload) + + # Controlled failure trigger for E5 + # if isinstance(stable_input, dict) and stable_input.get("force_error") is True: + # raise ValueError("Controlled E5 failure: forced error triggered") + + stable_input_hash = sha256_text(canonical_json(stable_input)) + + artifact_filename = f"{stable_input_hash}.json" + artifact_path = module_dir / artifact_filename + + artifact_content = { + "input_hash": stable_input_hash, + "received_payload": stable_input, + "message": "integration pilot executed successfully", + "module_name": MODULE_NAME, + "module_version": MODULE_VERSION, + "processing_profile": "normalized-v2", + } + + serialized_artifact = json.dumps( + artifact_content, + sort_keys=True, + indent=2, + ensure_ascii=False, + ) + artifact_path.write_text(serialized_artifact, encoding="utf-8") + + artifact_hash = sha256_text(serialized_artifact) + + return { + "module_name": MODULE_NAME, + "version": MODULE_VERSION, + "status": "success", + "output": { + "message": "pilot module executed successfully", + "input_hash": stable_input_hash, + "received_keys": sorted(list(stable_input.keys())) + if isinstance(stable_input, dict) + else [], + "processing_profile": "normalized-v2", + }, + "artifacts": [ + { + "module_name": MODULE_NAME, + "file_path": str(artifact_path), + "hash": artifact_hash, + "type": "pilot_output", + } + ], + } \ No newline at end of file diff --git a/modules/integration-pilot/app/main.py.bak b/modules/integration-pilot/app/main.py.bak new file mode 100644 index 0000000..a8e644f --- /dev/null +++ b/modules/integration-pilot/app/main.py.bak @@ -0,0 +1,84 @@ +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Dict + +from fastapi import FastAPI + +app = FastAPI(title="OpenPolicyStack Integration Pilot", version="0.1.0") + +MODULE_NAME = os.getenv("OPS_MODULE_NAME", "integration-pilot") +MODULE_VERSION = os.getenv("PILOT_MODULE_VERSION", "0.1.0") +ARTIFACT_ROOT = Path(os.getenv("OPS_ARTIFACT_ROOT", "/var/openpolicystack/artifacts")) + + +def canonical_json(value: Any) -> str: + """ + Deterministic JSON serialization used for stable content hashing. + """ + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +@app.get("/health") +def health() -> Dict[str, Any]: + return { + "status": "ok", + "module_name": MODULE_NAME, + "version": MODULE_VERSION, + } + + +@app.post("/execute") +def execute(payload: Dict[str, Any]) -> Dict[str, Any]: + module_dir = ARTIFACT_ROOT / MODULE_NAME + module_dir.mkdir(parents=True, exist_ok=True) + + stable_input = payload.get("input", payload) + stable_input_hash = sha256_text(canonical_json(stable_input)) + + artifact_filename = f"{stable_input_hash}.json" + artifact_path = module_dir / artifact_filename + + artifact_content = { + "input_hash": stable_input_hash, + "received_payload": stable_input, + "message": "integration pilot executed successfully", + "module_name": MODULE_NAME, + "module_version": MODULE_VERSION, + } + + serialized_artifact = json.dumps( + artifact_content, + sort_keys=True, + indent=2, + ensure_ascii=False, + ) + artifact_path.write_text(serialized_artifact, encoding="utf-8") + + artifact_hash = sha256_text(serialized_artifact) + + return { + "module_name": MODULE_NAME, + "version": MODULE_VERSION, + "status": "success", + "output": { + "message": "pilot module executed successfully", + "input_hash": stable_input_hash, + "received_keys": sorted(list(stable_input.keys())) + if isinstance(stable_input, dict) + else [], + }, + "artifacts": [ + { + "module_name": MODULE_NAME, + "file_path": str(artifact_path), + "hash": artifact_hash, + "type": "pilot_output", + } + ], + } \ No newline at end of file diff --git a/modules/integration-pilot/requirements.txt b/modules/integration-pilot/requirements.txt new file mode 100644 index 0000000..926ab65 --- /dev/null +++ b/modules/integration-pilot/requirements.txt @@ -0,0 +1,2 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.6 \ No newline at end of file diff --git a/modules/orchestrator/.env.example b/modules/orchestrator/.env.example new file mode 100644 index 0000000..bf21d5d --- /dev/null +++ b/modules/orchestrator/.env.example @@ -0,0 +1,2 @@ +OPS_ENV=dev +OPS_LOG_LEVEL=INFO \ No newline at end of file diff --git a/modules/orchestrator/COMPOSE_DERIVATION_MATRIX.md b/modules/orchestrator/COMPOSE_DERIVATION_MATRIX.md new file mode 100644 index 0000000..67f6270 --- /dev/null +++ b/modules/orchestrator/COMPOSE_DERIVATION_MATRIX.md @@ -0,0 +1,21 @@ +# OpenPolicyStack Compose Derivation Matrix + +**File:** `COMPOSE_DERIVATION_MATRIX.md` + +**Version:** 1.0 + +**Status:** Compose Derivation Matrix - MVP Draft + +--- + +| Service Name | Build Context | Image Name | Internal Port | Host Port | Networks | Volumes | Env Source | Healthcheck | Role / Notes | +| ------------------- | ----------------------------- | --------------------------------------- | ------------- | --------- | ---------- | -------------------------------------------------------------------------------------------- | ---------------------------------- | ------------- | ----------------------------------------------------------------------------------------- | +| `orchestrator` | `./orchestrator` | `openpolicystack/orchestrator:dev` | `8080` | `8100` | `ops-core` | `ops-artifacts:/var/openpolicystack/artifacts`, `ops-metadata:/var/openpolicystack/metadata` | `./orchestrator/.env` | `GET /health` | Central coordination service; invokes modules, records run metadata, persists SQLite data | +| `data-layer` | `./modules/data-layer` | `openpolicystack/data-layer:dev` | `8080` | — | `ops-core` | `ops-artifacts:/var/openpolicystack/artifacts` | `./modules/data-layer/.env` | `GET /health` | Provides data access / preprocessing service to workflows | +| `dr-anticorruption` | `./modules/dr-anticorruption` | `openpolicystack/dr-anticorruption:dev` | `8080` | — | `ops-core` | `ops-artifacts:/var/openpolicystack/artifacts` | `./modules/dr-anticorruption/.env` | `GET /health` | Domain analysis module integrated as black-box HTTP service | +| `monitor` | `./modules/monitor` | `openpolicystack/monitor:dev` | `8080` | — | `ops-core` | `ops-artifacts:/var/openpolicystack/artifacts` | `./modules/monitor/.env` | `GET /health` | Monitoring / tracking module integrated through standard module contract | +| `policy-simulator` | `./modules/policy-simulator` | `openpolicystack/policy-simulator:dev` | `8080` | — | `ops-core` | `ops-artifacts:/var/openpolicystack/artifacts` | `./modules/policy-simulator/.env` | `GET /health` | Simulation module invoked by deterministic workflow templates | +| `strategy-agent` | `./modules/strategy-agent` | `openpolicystack/strategy-agent:dev` | `8080` | — | `ops-core` | `ops-artifacts:/var/openpolicystack/artifacts` | `./modules/strategy-agent/.env` | `GET /health` | Strategy support module exposed as standardized REST service | +| `supplychain-risk` | `./modules/supplychain-risk` | `openpolicystack/supplychain-risk:dev` | `8080` | — | `ops-core` | `ops-artifacts:/var/openpolicystack/artifacts` | `./modules/supplychain-risk/.env` | `GET /health` | Risk analysis module integrated as independent microservice | + + diff --git a/modules/orchestrator/Dockerfile b/modules/orchestrator/Dockerfile new file mode 100644 index 0000000..7b7d624 --- /dev/null +++ b/modules/orchestrator/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app + +EXPOSE 8080 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/modules/orchestrator/INTEGRATION_SPEC.md b/modules/orchestrator/INTEGRATION_SPEC.md new file mode 100644 index 0000000..41ccf12 --- /dev/null +++ b/modules/orchestrator/INTEGRATION_SPEC.md @@ -0,0 +1,590 @@ +# OpenPolicyStack Integration Specification + +**File:** `INTEGRATION_SPEC.md` + +**Version:** 1.0 + +**Status:** Frozen Integration Contract + +--- + +# 1. Purpose + +This document defines the **integration contract governing all modules within the OpenPolicyStack environment**. + +The purpose of this specification is to ensure that independently developed analytical modules can be orchestrated in a **deterministic, reproducible, and traceable execution environment**. + +The integration layer standardizes conventions for: + +- service naming +- API interfaces +- port allocation +- environment configuration +- networking +- logging +- artifact management + +By enforcing these conventions, the OpenPolicyStack orchestrator can coordinate heterogeneous modules while preserving **run-level reproducibility, structured provenance, and auditability**. + +This document represents the **authoritative integration contract** against which all future integration decisions are evaluated. + +--- + +# 2. Architectural Context + +OpenPolicyStack implements a **centralized orchestration architecture** in which multiple analytical modules are coordinated by a single orchestrator service. + +Each module: + +- operates as an **independent microservice** +- exposes a **lightweight REST interface** +- runs inside a **Docker container** +- executes deterministically within a predefined workflow template. + +The orchestrator performs: + +- module sequencing +- metadata capture +- artifact tracking +- module version recording +- execution provenance reconstruction. + +The MVP system integrates six modules: + +- Data Layer +- DR Anticorruption +- Monitor +- Policy Simulator +- Strategy Agent +- Supply Chain Risk + +Modules are treated as **black-box services**, allowing their internal analytical logic to remain independent while the orchestration layer enforces system-level governance guarantees. + +--- + +# 3. Core Design Principles + +All integration decisions must preserve the following architectural principles. + +## Deterministic Execution + +Workflows follow predefined templates specifying module order and execution structure. + +## Reproducibility + +Identical inputs and module versions must produce structurally identical execution runs. + +## Traceability + +All module invocations, inputs, outputs, and artifacts must be recorded in structured metadata. + +## Modular Interoperability + +Modules interact exclusively through standardized HTTP interfaces. + +## Governance by Design + +Accountability and auditability are enforced at the orchestration layer rather than within module internals. + +--- + +# 4. Repository Structure + +All modules must follow the repository layout: + +``` +/opt/openpolicystack +│ +├── orchestrator +│ +├── modules +│ ├── data-layer +│ ├── dr-anticorruption +│ ├── monitor +│ ├── policy-simulator +│ ├── strategy-agent +│ └── supplychain-risk +│ +├── infrastructure +│ +├── artifacts +│ +└── compose.yaml +``` + +Each module must reside under: + +``` +modules/ +``` + +Module names must be unique within the system. + +--- + +# 5. Module Naming Convention + +All services use **lowercase kebab-case identifiers**. + +Examples: + +``` +data-layer +policy-simulator +strategy-agent +monitor +``` + +The module identifier must be used consistently across: + +- repository folder name +- Docker image name +- Docker Compose service name +- network hostname +- artifact directory +- log fields +- module metadata + +Example: + +``` +modules/policy-simulator +service: policy-simulator +image: openpolicystack/policy-simulator +hostname: policy-simulator +``` + +--- + +# 6. Containerization Requirements + +Every module must provide: + +``` +Dockerfile +.env.example +module.yaml +``` + +Modules must run as **standalone containers**. + +The container must start an HTTP service exposing the module API. + +--- + +# 7. Port Allocation Scheme + +### Internal container port + +All module APIs must listen on: + +``` +8080 +``` + +Optional auxiliary ports: + +``` +9090 metrics +8081 admin/debug +``` + +--- + +### Service communication + +Services communicate via Docker DNS using service names: + +``` +http://policy-simulator:8080 +http://data-layer:8080 +``` + +Modules must **not rely on static IP addresses**. + +--- + +### Host port exposure + +Host ports are used only for development or gateway access. + +Reserved host port ranges: + +| Module | Port Range | +| --- | --- | +| orchestrator | 8100–8109 | +| data-layer | 8200–8209 | +| strategy-agent | 8300–8309 | +| dr-anticorruption | 8400–8409 | +| supplychain-risk | 8500–8509 | +| policy-simulator | 8600–8609 | +| monitor | 8700–8709 | + +--- + +# 8. API Integration Contract + +Modules must expose a **REST API interface**. + +### Required endpoints + +``` +GET /health +GET /metadata +POST /execute +``` + +These endpoints form the minimal integration surface. + +--- + +# 8.1 Health Endpoint + +``` +GET /health +``` + +Response: + +```json +{ + "status": "ok", + "module_name": "policy-simulator", + "version": "0.1.0" +} +``` + +Used for orchestrator readiness checks. + +--- + +# 8.2 Metadata Endpoint + +``` +GET /metadata +``` + +Example response: + +```json +{ + "module_name": "policy-simulator", + "version": "0.1.0", + "api_version": "1.0", + "owner": "module_author", + "supported_tasks": [ + "simulate_policy" + ] +} +``` + +The orchestrator records module version metadata for reproducibility tracking. + +--- + +# 8.3 Execution Endpoint + +``` +POST /execute +``` + +Request payload: + +```json +{ + "run_id": "uuid", + "parameters": {}, + "inputs": [], + "metadata": {} +} +``` + +Response payload must include: + +```json +{ + "module_name": "policy-simulator", + "version": "0.1.0", + "status": "success", + "output": {}, + "artifacts": [] +} +``` + +Required response fields: + +- module_name +- version +- status +- output +- artifacts + +This structure allows the orchestrator to capture module-level provenance. + +--- + +# 9. Environment Variable Conventions + +Global environment variables use the prefix: + +``` +OPS_ +``` + +Required base variables: + +``` +OPS_ENV +OPS_MODULE_NAME +OPS_PORT +OPS_LOG_LEVEL +OPS_ARTIFACT_ROOT +OPS_ORCHESTRATOR_URL +``` + +Example: + +``` +OPS_MODULE_NAME=policy-simulator +OPS_PORT=8080 +OPS_ARTIFACT_ROOT=/var/openpolicystack/artifacts +``` + +--- + +### Module-specific variables + +Module-specific variables must use a namespace: + +``` +MODULEPREFIX__ +``` + +Example: + +``` +POLICY_SIMULATOR__MODEL_PATH +DATA_LAYER__DB_PATH +``` + +--- + +# 10. Networking Model + +The deployment uses two Docker networks. + +### Internal network + +``` +ops-core +``` + +All modules and the orchestrator join this network. + +--- + +### Edge network + +``` +ops-edge +``` + +Used only for services exposed externally. + +Service discovery uses DNS names: + +``` +http://module-name:8080 +``` + +--- + +# 11. Logging Convention + +All services log to: + +``` +stdout +stderr +``` + +Logs must follow **structured JSON format**. + +Example: + +```json +{ + "timestamp": "2026-03-12T11:20:31Z", + "level": "INFO", + "service": "policy-simulator", + "run_id": "uuid", + "event": "simulation_started", + "message": "Policy simulation initiated" +} +``` + +Required fields: + +- timestamp +- level +- service +- run_id +- event +- message + +--- + +# 12. Artifact Management + +Artifacts are stored in a shared volume mounted at: + +``` +/var/openpolicystack/artifacts +``` + +Run directory structure: + +``` +artifacts/ +└── runs/ + └── / + └── / + ├── inputs/ + ├── outputs/ + └── meta/ +``` + +Example: + +``` +artifacts/runs/abc123/policy-simulator/outputs/report.json +``` + +Large outputs must be stored as artifacts with references returned to the orchestrator. + +The orchestrator records: + +- artifact path +- artifact hash +- producing module +- associated run + +This enables integrity verification and reproducibility validation. + +--- + +# 13. Module Manifest + +Each module must provide: + +``` +modules//module.yaml +``` + +Example: + +```yaml +module_name: policy-simulator +version: 0.1.0 + +interface: + type: http + port: 8080 + execute: /execute + health: /health +``` + +This allows automated module discovery and integration validation. + +--- + +# 14. Minimum Compliance Checklist + +A module is considered **integration-ready** only if it satisfies: + +- containerized via Docker +- resides under `modules/` +- exposes `/health` +- exposes `/metadata` +- exposes `/execute` +- accepts `run_id` +- logs structured JSON +- returns module version +- stores outputs as artifacts +- includes `module.yaml` +- includes `.env.example` + +--- + +# 15. Scope Boundaries + +The integration specification intentionally excludes: + +- distributed orchestration platforms +- ontology harmonization across modules +- semantic schema alignment +- adaptive runtime planning +- parallel execution scheduling + +These elements are outside the MVP scope and may be explored in future research iterations. + +--- + +# 16. Governance Role of the Orchestrator + +Modules remain responsible for **analytical correctness**. + +The orchestrator is responsible for: + +- workflow coordination +- version capture +- metadata persistence +- artifact lineage tracking +- reproducibility guarantees +- execution trace reconstruction. + +This separation ensures governance guarantees are **architectural system properties rather than implementation details of individual modules**. + +--- + +# 17. Change Management + +This specification represents the **baseline integration contract**. + +Changes must follow: + +``` +proposal → review → version update +``` + +Breaking interface changes require: + +``` +spec version increment +``` + +--- + +# 18. Relationship to Deployment Architecture + +This specification is the **foundation for the deployment architecture**. + +The Docker Compose infrastructure will implement the conventions defined here, including: + +- service names +- network topology +- environment variable schema +- artifact mounts +- inter-service communication + +--- + +# 19. Next Integration Step + +After freezing this specification, the integration process proceeds with: + +1. Generate the **Compose deployment skeleton** +2. Integrate the **first pilot module** +3. Validate compliance +4. Refine integration procedures +5. Expand to the full module ecosystem \ No newline at end of file diff --git a/modules/orchestrator/MODULE_INTEGRATION_GUIDE.md b/modules/orchestrator/MODULE_INTEGRATION_GUIDE.md new file mode 100644 index 0000000..55c2e4c --- /dev/null +++ b/modules/orchestrator/MODULE_INTEGRATION_GUIDE.md @@ -0,0 +1,472 @@ +# OpenPolicyStack Module Integration Guide + +**File:** `MODULE_INTEGRATION_GUIDE.md` + +**Version:** 1.0 + +**Status:** Frozen Module Integration Guide + +--- + +# 1. Overview + +This guide explains how to prepare your module so that it can be integrated into the **OpenPolicyStack Orchestration System**. + +All modules in OpenPolicyStack operate as **independent containerized** microservices coordinated by a **central** orchestrator. + +Your module will: + +- Run inside a Docker container. +- Expose a lightweight REST API. +- Receive execution requests from the orchestrator. +- Return structured outputs and artifact references. + +You **do not** need to implement orchestration logic**.** + +Your module must simply **conform** to the Integration Contract defined in this guide. + +--- + +# 2. Validated Integration Baseline + +A working integration baseline has been implemented and validated. + +The module: +``` +modules/integration-pilot +``` +Serves as the reference implementation for: + +- API contract +- Artifact handling +- Environment variables +- Docker container behavior +- Orchestrator interaction + +All modules must align with the integration-pilot pattern. +If in doubt, follow that implementation exactly. + +--- + +# 3. High-Level Integration Flow + +When OpenPolicyStack runs a workflow, the following occurs: + +1. The orchestrator generates a **run_id**. +2. The orchestrator calls your module via HTTP. +3. Your module performs its computation. +4. Your module returns structured JSON. +5. Artifacts are written to the shared volume. +6. The orchestrator records execution metadata. + +Your module remains **analytically independent** but participates in the **shared execution environment**. + +--- + +# 4. Module Repository Structure + +Your module must follow this directory structure: + +``` +modules// +│ +├── app/ +│ └── main.py +│ +├── Dockerfile +├── requirements.txt +├── module.yaml +├── .env.example +└── README.md +``` + +--- + +# 5. Naming Rules + +Module names must follow **lowercase kebab-case**. + +Correct: + +``` +policy-simulator +strategy-agent +supplychain-risk +data-layer +``` + +Incorrect: + +``` +PolicySimulator +policy_simulator +policySimulator +``` + +Your module name must match: + +- Folder name +- Docker service name +- Container hostname +- Artifact directory name + +--- + +# 6. Docker Container Requirement + +Your module **must run inside a Docker container**. + +Validated minimal Dockerfile: + +``` +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app + +EXPOSE 8080 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] +``` + +The container must start a web server exposing the module API. + +--- + +# 7. API Interface Requirements + +Each module must expose three HTTP endpoints. + +### Required endpoints + +``` +GET /health +GET /metadata +POST /execute +``` + +All endpoints must return JSON. + +--- + +# 7.1 Health Endpoint + +Used by the orchestrator to verify that your service is running. + +``` +GET /health +``` +Must: +- Return HTTP 200. +- Respond quickly (no heavy logic). +- Not depend on external services. + + +Example response: + +``` +{ + "status":"ok", + "module_name":"policy-simulator", + "version":"0.1.0" +} +``` +This endpoint is used for: +- Container healthchecks +- Orchestration readiness + +--- + +# 7.2 Metadata Endpoint + +Provides module information used for orchestration and debugging. + +``` +GET /metadata +``` + +--- + +# 7.3 Execute Endpoint + +This is the **main entry point** used by the orchestrator. + +``` +POST /execute +``` +Your module must: +- Accept JSON input. +- Return structured JSON output. +- Never return raw text or HTML. + +### Required Response Structure + +``` +{ + "module_name": "policy-simulator", + "version": "0.1.0", + "status": "success", + "output": {}, + "artifacts": [] +} +``` +### Required Response Fields + +| Field | Description | +| ----------- | --------------------------- | +| module_name | name of the module | +| version | module version | +| status | success or failure | +| output | structured JSON result | +| artifacts | list of artifact references | + +--- + +# 8. Service Port + +All modules must listen on: + +``` +8080 +``` + +--- + +# 9. Environment Variables + +Each module must include: + +``` +.env.example +``` +### Important Rule +- .env.example → committed to Git + +- .env → NOT committed (local runtime only) + +Each developer must create: + +``` +cp .env.example .env +``` + +### Important +Add the following at the bottom of your .gitignore to allow Git to track your .env.example file. + +``` +# Allow example env templates (must be AFTER any .env / .env.* ignores) +!.env.example +!**/.env.example +``` + +### Example env.example + +``` +OPS_ENV=dev +OPS_LOG_LEVEL=INFO +OPS_PORT=8080 +OPS_ARTIFACT_ROOT=/var/openpolicystack/artifacts +OPS_MODULE_NAME=policy-simulator +``` +### Why This Matters + +This prevents; +- Missing environment variables in deployment. +- Inconsistent runtime configuration. +- Integration failures on the VM. + +--- + +# 10. Artifact Storage (Validated Behavior) + +Artifacts must be written to: + +``` +/var/openpolicystack/artifacts +``` + +Each module must create its own subdirectory: + +``` +/var/openpolicystack/artifacts// +``` + +Example: + +``` +/var/openpolicystack/artifacts/integration-pilot/.json +``` + +--- + +### Artifact Response Format + +``` +{ + "artifacts": [ + { + "module_name":"policy-simulator", + "file_path":"/var/openpolicystack/artifacts/policy-simulator/output.json", + "type":"output" + } + ] +} +``` + +--- + +### Important + +- The orchestrator reads **references**, not raw files. +- All containers share the same mounted volume. +- Do not use local paths outside `/var/openpolicystack/artifacts`. + +--- + +# 11. Logging + +Modules must log to: + +``` +stdout / stderr +``` +Use structured JSON logs. + +--- + +# 12. Module Manifest + +Each module must include a `module.yaml`. + +``` +module_name: policy-simulator +version: 0.1.0 + +interface: + type: http + port: 8080 + health: /health + metadata: /metadata + execute: /execute +``` + +--- + +# 13. Local Testing +Before integration: + +``` +docker build -t openpolicystack/ . +docker run -p 8080:8080 openpolicystack/ +``` + +Test endpoint: + +``` +curl http://localhost:8080/health +``` + +--- + +# 14. Integration Testing (Validated Workflow) +Once integrated with the orchestrator: + +``` +docker compose build +docker compose up-d +docker composeps +``` + +Test orchestrator: + +``` +curl http://localhost:8100/health +``` + +Test execution: + +``` +curl-X POST http://localhost:8100/execute \ +-H"Content-Type: application/json" \ +-d'{"test":"hello"}' +``` + +--- + + +# 15. Integration Checklist + +Before submitting your module for integration, verify: + +✔ Docker builds successfully. + +✔ Service runs on port 8080. + +✔ /health returns 200. + +✔ /execute returns valid JSON. + +✔ .env.example included. + +✔ artifacts written to shared volume. + +✔ response includes artifacts field. + +✔ module follows integration-pilot pattern. + +--- + +# 16. Common Mistakes +### Using localhost for inter-service calls + +Wrong: + +``` +http://localhost:8080 +``` + +Correct: + +``` +http://orchestrator:8080 +``` + +--- + +### Missing `.env.example` + +This causes deployment failures on the VM. + +--- + +### Health endpoint too slow + +Healthchecks will fail → container marked unhealthy. + +--- + +### Writing artifacts outside shared volume + +Artifacts will not be visible to other services. + +# 17. Final Note + +The integration-pilot module is the single source of truth for: + +- Correct module behavior. + +- Correct response structure. + +- Correct artifact handling. + +- Correct environment configuration. + +If your module behaves differently, it will fail integration. + diff --git a/modules/orchestrator/README.md b/modules/orchestrator/README.md index 601f6de..5469c19 100644 --- a/modules/orchestrator/README.md +++ b/modules/orchestrator/README.md @@ -1,4 +1,186 @@ # Orchestrator -Owner: Adrián Con -Goal: /plan_and_execute + reproducible runs + traces/logging. +- **Owner:** Adrián Con García +- **Module Path:** `modules/orchestrator` +- **Goal:** Provide a central orchestrator service that calls OpenPolicyStack microservices and produces reproducible “runs” with basic logs/traces. + +--- + +## 1) Scope (What This Module Does) + +The **Orchestrator** coordinates end-to-end workflows across OpenPolicyStack modules. + +It is responsible for: +- Receiving a **Scenario Request** (what to run + which modules + parameters). +- Creating a **Reproducible Run Record** (e.g: Inputs, timestamps, versions, outputs). +- Executing a workflow via a single entrypoint: **`/plan_and_execute`**. +- Calling module microservices (HTTP) and collecting their outputs/artifacts. +- Writing basic **Logs** and optional **Trace Data** for transparency/debugging. +- Returning a structured response that can be plugged into a demo portal/workflow. + +**Non-Goals (MVP / Scope NOW)** +- Not an API Gateway or Portal (handled by the **Integration UI / API Gateway** effort). +- Not advanced **Monitoring & Telemetry** (phase 2). +- Not **Privacy & PII Redaction** (phase 2 / optional later). +- Not complex workflow engines (queues, distributed scheduling, etc.); keep sequential execution. +- Not implementing the analytics logic of other modules. +Out of scope for NOW; may be integrated later as separate services. + +--- + +## 2) Definitions (Plain Language) +- **Run:** One execution of the system for a given scenario + parameters. +- **Run ID:** Unique identifier for a run (used to find outputs/logs later). +- **Plan:** A list of steps the orchestrator will execute (e.g: call policy simulator → call strategy agent). +- **Artifact:** A saved output file (plot image, brief markdown, JSON results, etc.). +- **Trace/Logs:** A record of what happened during a run (steps, timing, success/failure). + +--- + +## 3) Interface (What This Module Exposes) + +### MVP API (Proposed Interface) +The orchestrator exposes a small HTTP API. + +#### `POST /plan_and_execute` +Create a run, generate a simple plan (sequence of module calls), execute it, and persist inputs/outputs/logs under a `run_id`. + +**Request (example):** This request starts a new orchestrated run for a demo policy scenario. It specifies which modules to execute and provides the scenario inputs (e.g: country, time horizon, and policy parameters like a VAT rate change). +```json +{ + "scenario_id": "demo-scenario-001", + "modules": ["policy-simulator", "strategy-agent"], + "inputs": { + "country": "DR", + "time_horizon_years": 5, + "policy_parameters": { + "vat_rate": 0.18 + } + }, + "run_options": { + "seed": 42, + "save_artifacts": true + } +} +``` + + +**Response (example):** The orchestrator returns a unique `run_id`, the execution plan it followed, and pointers to the saved outputs (KPIs/plots/brief) plus where logs and artifacts were stored for reproducibility. + +```json +{ + "run_id": "run_2026-01-09T12-34-56Z_ab12cd", + "status": "completed", + "plan": [ + {"step": 1, "module": "policy-simulator", "action": "execute"}, + {"step": 2, "module": "strategy-agent", "action": "execute"} + ], + "results": { + "policy-simulator": {"kpis": {"gdp_growth": 0.02}, "artifacts": ["kpis.json", "plot.png"]}, + "strategy-agent": {"brief": "brief.md", "artifacts": ["brief.md"]} + }, + "artifacts_path": "runs/run_2026-01-09T12-34-56Z_ab12cd/", + "logs_path": "runs/run_2026-01-09T12-34-56Z_ab12cd/logs.jsonl" +} +``` +`GET /runs/{run_id}` + +Returns the saved run metadata (inputs, plan, results pointers). + +`GET /health` + +Simple healthcheck endpoint. + +## 4) Inputs → Outputs (MVP) + +### Inputs +- `scenario_id` (string) +- `modules` (list of module names to call) +- `inputs` (JSON payload passed to modules) +- optional `run_options` (seed, toggles for saving artifacts, etc.) + +### Outputs +- `run_id` +- `plan` (what steps were executed) +- per-module results (JSON + artifact references) +- paths/pointers to stored logs and artifacts for reproducibility + +## 5) Run Storage (Reproducibility) + +Every call to `POST /plan_and_execute` creates a **run**. +A run is a single execution of a scenario with specific modules + inputs. + +To make runs **reproducible and traceable**, the orchestrator saves: +- The original request (inputs, chosen modules, options like seed), +- What steps were executed (the “plan”), +- Each module’s returned results, +- Basic logs of what happened. + +This is stored under a unique `run_id` in a folder like: + + +
runs/<run_id>/
+  run.json        # input request + derived plan + timestamps
+  results.json    # merged results (pointers to artifacts)
+  logs.jsonl      # structured logs (one JSON per line)
+  artifacts/
+    <module-name>/
+      ...         # plots, JSONs, briefs, etc.
+ +The API response includes the `run_id` and (optionally) paths/pointers to the saved logs and artifacts. + +## 6) How Modules Are Called (Assumptions for Integration) + +For MVP integration, the orchestrator assumes modules are reachable over **HTTP** (Docker-first) and expose a minimal interface so they can be called consistently. + +The proposed minimal module contract is documented in the repo Issue: +**“MVP module interface contract (proposed)”** (see GitHub Issues). + +In short, the orchestrator expects: +- `GET /health` for basic service readiness checks +- One primary execution endpoint (preferred: `POST /execute`, or a module-specific endpoint such as `/score`, `/risk`, `/run_scenario`) +- JSON-in / JSON-out +- A consistent response “envelope” including: + - `status`, `module`, `outputs` + - Optional `artifacts` and `evidence` fields + +> Note: Endpoint naming and exact payload fields may be refined during integration month. The orchestrator will prioritize compatibility with the agreed contract in the Issue and adapt via lightweight adapters if needed. + +## 7) How To Run (Local / Docker) + +> Status: This module is currently in setup phase. The commands below describe the intended MVP run method and will be made runnable as the skeleton is implemented. + +### Docker (recommended for reproducibility) +Planned workflow: +```bash +# from modules/orchestrator +docker build -t openpolicystack-orchestrator:dev . +docker run --rm -p 8000:8000 \ + -v $(pwd)/runs:/app/runs \ + openpolicystack-orchestrator:dev +``` + +### Local (for development) +Planned workflow: +```bash +# from modules/orchestrator +pip install -r requirements.txt +# example server command (framework to be confirmed) +python -m src.main +``` +## 8) Quickstart Demo (Planned) + +> Status: This is the intended MVP smoke test once the orchestrator skeleton is implemented. + +1) Start the orchestrator (see Section 7) +2) Start at least one module service (or a stub) reachable by the orchestrator +3) Trigger a run: + +```bash +curl -X POST http://localhost:8000/plan_and_execute \ + -H "Content-Type: application/json" \ + -d @examples/plan_and_execute.request.json +``` +Expected behavior (once implemented): +- Returns a JSON response containing a `run_id` +- Creates `runs//` with `run.json`, `results.json`, `logs.jsonl`, and `artifacts/` diff --git a/modules/orchestrator/app/__init__.py b/modules/orchestrator/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/orchestrator/app/main.py b/modules/orchestrator/app/main.py new file mode 100644 index 0000000..90def85 --- /dev/null +++ b/modules/orchestrator/app/main.py @@ -0,0 +1,518 @@ +import hashlib +import json +import os +import sqlite3 +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +import httpx +from fastapi import FastAPI, HTTPException + +app = FastAPI(title="OpenPolicyStack Orchestrator", version="0.1.0") + +MODULE_NAME = os.getenv("OPS_MODULE_NAME", "orchestrator") +MODULE_VERSION = "0.1.0" +ARTIFACT_ROOT = Path(os.getenv("OPS_ARTIFACT_ROOT", "/var/openpolicystack/artifacts")) +SQLITE_PATH = Path( + os.getenv( + "ORCHESTRATOR__SQLITE_PATH", + "/var/openpolicystack/metadata/orchestrator.db", + ) +) +INTEGRATION_PILOT_URL = os.getenv( + "ORCHESTRATOR__INTEGRATION_PILOT_URL", + "http://integration-pilot:8080", +) + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def ensure_paths() -> None: + ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True) + SQLITE_PATH.parent.mkdir(parents=True, exist_ok=True) + + +def get_conn() -> sqlite3.Connection: + ensure_paths() + conn = sqlite3.connect(SQLITE_PATH) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON;") + return conn + + +def canonical_json(value: Any) -> str: + """ + Deterministic JSON serialization for hashing and storage consistency. + """ + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def hash_payload(payload: Optional[Dict[str, Any]]) -> Optional[str]: + if payload is None: + return None + return sha256_text(canonical_json(payload)) + + +def hash_file(file_path: Path) -> str: + digest = hashlib.sha256() + with file_path.open("rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + digest.update(chunk) + return digest.hexdigest() + + +def column_exists(conn: sqlite3.Connection, table_name: str, column_name: str) -> bool: + rows = conn.execute(f"PRAGMA table_info({table_name})").fetchall() + return any(row[1] == column_name for row in rows) + + +def init_db() -> None: + with get_conn() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS runs ( + run_id TEXT PRIMARY KEY, + workflow_template TEXT NOT NULL, + overall_status TEXT NOT NULL, + created_at TEXT NOT NULL, + completed_at TEXT, + request_payload TEXT, + response_payload TEXT + ) + """ + ) + + conn.execute( + """ + CREATE TABLE IF NOT EXISTS module_calls ( + call_id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + module_name TEXT NOT NULL, + module_version TEXT, + call_sequence INTEGER NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + completed_at TEXT, + execution_time_ms INTEGER, + request_payload TEXT, + response_payload TEXT, + error_message TEXT, + FOREIGN KEY (run_id) REFERENCES runs(run_id) + ) + """ + ) + + conn.execute( + """ + CREATE TABLE IF NOT EXISTS artifacts ( + artifact_id TEXT PRIMARY KEY, + run_id TEXT NOT NULL, + call_id TEXT, + module_name TEXT NOT NULL, + artifact_type TEXT, + file_path TEXT NOT NULL, + hash TEXT, + created_at TEXT NOT NULL, + FOREIGN KEY (run_id) REFERENCES runs(run_id), + FOREIGN KEY (call_id) REFERENCES module_calls(call_id) + ) + """ + ) + + # Safe additive migration for hashing support + if not column_exists(conn, "runs", "request_payload_hash"): + conn.execute("ALTER TABLE runs ADD COLUMN request_payload_hash TEXT") + + if not column_exists(conn, "runs", "response_payload_hash"): + conn.execute("ALTER TABLE runs ADD COLUMN response_payload_hash TEXT") + + if not column_exists(conn, "module_calls", "request_payload_hash"): + conn.execute("ALTER TABLE module_calls ADD COLUMN request_payload_hash TEXT") + + if not column_exists(conn, "module_calls", "response_payload_hash"): + conn.execute("ALTER TABLE module_calls ADD COLUMN response_payload_hash TEXT") + + conn.commit() + + +def insert_run( + run_id: str, + workflow_template: str, + overall_status: str, + created_at: str, + request_payload: Dict[str, Any], +) -> None: + request_payload_json = canonical_json(request_payload) + request_payload_hash = sha256_text(request_payload_json) + + with get_conn() as conn: + conn.execute( + """ + INSERT INTO runs ( + run_id, + workflow_template, + overall_status, + created_at, + request_payload, + response_payload, + request_payload_hash, + response_payload_hash + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + run_id, + workflow_template, + overall_status, + created_at, + request_payload_json, + None, + request_payload_hash, + None, + ), + ) + conn.commit() + + +def update_run( + run_id: str, + overall_status: str, + response_payload: Dict[str, Any], + completed_at: Optional[str] = None, +) -> None: + response_payload_json = canonical_json(response_payload) + response_payload_hash = sha256_text(response_payload_json) + + with get_conn() as conn: + conn.execute( + """ + UPDATE runs + SET overall_status = ?, + completed_at = ?, + response_payload = ?, + response_payload_hash = ? + WHERE run_id = ? + """, + ( + overall_status, + completed_at, + response_payload_json, + response_payload_hash, + run_id, + ), + ) + conn.commit() + + +def insert_module_call( + call_id: str, + run_id: str, + module_name: str, + call_sequence: int, + status: str, + started_at: str, + request_payload: Dict[str, Any], +) -> None: + request_payload_json = canonical_json(request_payload) + request_payload_hash = sha256_text(request_payload_json) + + with get_conn() as conn: + conn.execute( + """ + INSERT INTO module_calls ( + call_id, + run_id, + module_name, + module_version, + call_sequence, + status, + started_at, + completed_at, + execution_time_ms, + request_payload, + response_payload, + error_message, + request_payload_hash, + response_payload_hash + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + call_id, + run_id, + module_name, + None, + call_sequence, + status, + started_at, + None, + None, + request_payload_json, + None, + None, + request_payload_hash, + None, + ), + ) + conn.commit() + + +def update_module_call( + call_id: str, + module_version: Optional[str], + status: str, + completed_at: str, + execution_time_ms: int, + response_payload: Optional[Dict[str, Any]] = None, + error_message: Optional[str] = None, +) -> None: + response_payload_json = ( + canonical_json(response_payload) if response_payload is not None else None + ) + response_payload_hash = ( + sha256_text(response_payload_json) if response_payload_json is not None else None + ) + + with get_conn() as conn: + conn.execute( + """ + UPDATE module_calls + SET module_version = ?, + status = ?, + completed_at = ?, + execution_time_ms = ?, + response_payload = ?, + error_message = ?, + response_payload_hash = ? + WHERE call_id = ? + """, + ( + module_version, + status, + completed_at, + execution_time_ms, + response_payload_json, + error_message, + response_payload_hash, + call_id, + ), + ) + conn.commit() + + +def insert_artifact( + artifact_id: str, + run_id: str, + call_id: Optional[str], + module_name: str, + artifact_type: Optional[str], + file_path: str, + hash_value: Optional[str], + created_at: str, +) -> None: + with get_conn() as conn: + conn.execute( + """ + INSERT INTO artifacts ( + artifact_id, + run_id, + call_id, + module_name, + artifact_type, + file_path, + hash, + created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + artifact_id, + run_id, + call_id, + module_name, + artifact_type, + file_path, + hash_value, + created_at, + ), + ) + conn.commit() + + +@app.on_event("startup") +def startup_event() -> None: + init_db() + + +@app.get("/health") +def health() -> Dict[str, Any]: + return { + "status": "ok", + "module_name": MODULE_NAME, + "version": MODULE_VERSION, + "pilot_url": INTEGRATION_PILOT_URL, + "sqlite_path": str(SQLITE_PATH), + } + + +@app.post("/execute") +async def execute(payload: Dict[str, Any]) -> Dict[str, Any]: + run_id = str(uuid.uuid4()) + run_created_at = now_iso() + + insert_run( + run_id=run_id, + workflow_template="pilot-workflow", + overall_status="running", + created_at=run_created_at, + request_payload=payload, + ) + + call_id = str(uuid.uuid4()) + call_started_at = now_iso() + + upstream_payload = { + "run_id": run_id, + "input": payload, + "requested_by": MODULE_NAME, + } + + insert_module_call( + call_id=call_id, + run_id=run_id, + module_name="integration-pilot", + call_sequence=1, + status="running", + started_at=call_started_at, + request_payload=upstream_payload, + ) + + try: + start_perf = time.perf_counter() + + async with httpx.AsyncClient(timeout=30.0) as client: + upstream_response = await client.post( + f"{INTEGRATION_PILOT_URL}/execute", + json=upstream_payload, + ) + upstream_response.raise_for_status() + pilot_result = upstream_response.json() + + execution_time_ms = int((time.perf_counter() - start_perf) * 1000) + call_completed_at = now_iso() + + update_module_call( + call_id=call_id, + module_version=pilot_result.get("version"), + status="success", + completed_at=call_completed_at, + execution_time_ms=execution_time_ms, + response_payload=pilot_result, + ) + + except Exception as exc: + execution_time_ms = ( + int((time.perf_counter() - start_perf) * 1000) + if "start_perf" in locals() + else 0 + ) + call_completed_at = now_iso() + run_completed_at = now_iso() + + update_module_call( + call_id=call_id, + module_version=None, + status="failed", + completed_at=call_completed_at, + execution_time_ms=execution_time_ms, + response_payload=None, + error_message=str(exc), + ) + + error_response = { + "module_name": MODULE_NAME, + "version": MODULE_VERSION, + "status": "failed", + "run_id": run_id, + "error": f"Pilot module call failed: {exc}", + } + + update_run( + run_id=run_id, + overall_status="failed", + response_payload=error_response, + completed_at=run_completed_at, + ) + + raise HTTPException(status_code=502, detail=f"Pilot module call failed: {exc}") + + for artifact in pilot_result.get("artifacts", []): + insert_artifact( + artifact_id=str(uuid.uuid4()), + run_id=run_id, + call_id=call_id, + module_name=artifact.get("module_name", "integration-pilot"), + artifact_type=artifact.get("type"), + file_path=artifact.get("file_path", ""), + hash_value=artifact.get("hash"), + created_at=now_iso(), + ) + + orchestrator_dir = ARTIFACT_ROOT / "orchestrator" + orchestrator_dir.mkdir(parents=True, exist_ok=True) + + summary_path = orchestrator_dir / f"{run_id}-summary.json" + summary = { + "run_id": run_id, + "timestamp": run_created_at, + "orchestrator": MODULE_NAME, + "pilot_response": pilot_result, + } + summary_serialized = json.dumps(summary, indent=2, ensure_ascii=False) + summary_path.write_text(summary_serialized, encoding="utf-8") + summary_hash = hash_file(summary_path) + + insert_artifact( + artifact_id=str(uuid.uuid4()), + run_id=run_id, + call_id=None, + module_name=MODULE_NAME, + artifact_type="run_summary", + file_path=str(summary_path), + hash_value=summary_hash, + created_at=now_iso(), + ) + + final_response = { + "module_name": MODULE_NAME, + "version": MODULE_VERSION, + "status": "success", + "run_id": run_id, + "output": { + "message": "orchestrator executed pilot workflow successfully", + "pilot_result": pilot_result, + }, + "artifacts": [ + { + "module_name": MODULE_NAME, + "file_path": str(summary_path), + "hash": summary_hash, + "type": "run_summary", + } + ], + } + + update_run( + run_id=run_id, + overall_status="success", + response_payload=final_response, + completed_at=now_iso(), + ) + + return final_response \ No newline at end of file diff --git a/modules/orchestrator/requirements.txt b/modules/orchestrator/requirements.txt new file mode 100644 index 0000000..8a89da7 --- /dev/null +++ b/modules/orchestrator/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +httpx==0.27.2 \ No newline at end of file