From 01e95c6a0452a48cafe6e8bdaae1989ddeb17486 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Wed, 15 Jul 2026 08:43:10 +0300 Subject: [PATCH 01/10] fix race bug --- docs/services.md | 11 + src/components/c2d/compute_engine_docker.ts | 82 +++++- src/test/integration/services.test.ts | 84 +++++++ src/test/unit/compute.test.ts | 4 +- .../service/serviceNetworkCleanup.test.ts | 1 + .../unit/service/serviceRestartRace.test.ts | 237 ++++++++++++++++++ 6 files changed, 409 insertions(+), 10 deletions(-) create mode 100644 src/test/unit/service/serviceRestartRace.test.ts diff --git a/docs/services.md b/docs/services.md index 6c8619b9c..6e2d49d7f 100644 --- a/docs/services.md +++ b/docs/services.md @@ -84,6 +84,17 @@ the network always reflects the current code's configuration instead of silently whatever options a previous node version created it with, and it matches restart's overall tear-down-and-rebuild semantics (the container is never reused either). +**Lifecycle operations are exclusive per service.** At most one lifecycle operation — the +background start pipeline, `SERVICE_RESTART`, `SERVICE_STOP`, or the expiry sweep — runs per +service at a time. A restart or stop issued while another operation is in flight (e.g. a +restart still pulling the image) is rejected with +`Service has a start/stop/restart operation in progress — retry shortly`; simply retry +once the in-flight operation settles. Without this exclusivity, the background loop's +crash-orphan recovery could tear down the `ocean-svc-` network in the middle of a +restart that had just created it, failing the restart with +`network ocean-svc- not found`. If a service expires while such an operation is in +flight, the expiry sweep simply retries on a later tick. + ## Configuration Service-on-demand is configured per Docker connection under `serviceOnDemand`: diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 36f14e893..b1f3857a3 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -104,9 +104,14 @@ export class C2DEngineDocker extends C2DEngine { private stopped: boolean = false // The currently-running InternalLoop pass, so stop() can drain it before returning. private internalLoopPromise: Promise | null = null - // serviceIds currently being advanced by processServiceStart, so the InternalLoop doesn't - // launch a second pipeline for the same service while one is already in flight. - private servicesBeingStarted: Set = new Set() + // Per-service lifecycle lock: serviceIds with an exclusive operation in flight — the + // loop-driven start pipeline (processServiceStart), restartService or stopService. At + // most one such operation may run per service: the InternalLoop skips locked ids + // (including its expiry sweep), restart/stop throw. Without this, the loop's + // orphan-recovery tears down the network a concurrent restart just created (the + // "network ocean-svc- not found" failure) and start/stop/restart clobber each + // other's docker resources and job status. + private serviceOpsInFlight: Set = new Set() // The in-flight processServiceStart() promises (launched fire-and-forget by InternalLoop), // so stop() can drain them before returning — otherwise a start could outlive stop() and // race a restarted engine on the same shared DB. @@ -1787,11 +1792,11 @@ export class C2DEngineDocker extends C2DEngine { this.getC2DConfig().hash ) for (const svc of pendingStarts) { - if (this.servicesBeingStarted.has(svc.serviceId)) continue - this.servicesBeingStarted.add(svc.serviceId) + if (this.serviceOpsInFlight.has(svc.serviceId)) continue + this.serviceOpsInFlight.add(svc.serviceId) // Track the promise so stop() can drain it; clean both trackers when it settles. const startPromise = this.processServiceStart(svc).finally(() => { - this.servicesBeingStarted.delete(svc.serviceId) + this.serviceOpsInFlight.delete(svc.serviceId) this.serviceStartPromises.delete(startPromise) }) this.serviceStartPromises.add(startPromise) @@ -1803,11 +1808,17 @@ export class C2DEngineDocker extends C2DEngine { ) for (const svc of expiredServices) { CORE_LOGGER.info(`Service ${svc.serviceId} expired — stopping`) - await this.stopService(svc.serviceId, svc.owner).catch((e) => { + try { + await this.stopService(svc.serviceId, svc.owner) + } catch (e: any) { + // Typically the lifecycle lock (a restart/stop already in flight). Marking + // Expired without teardown would leak the container/ports, so defer the whole + // sweep for this service to the next tick. CORE_LOGGER.error( - `Failed to stop expired service ${svc.serviceId}: ${e.message}` + `Failed to stop expired service ${svc.serviceId}: ${e.message} — retrying next tick` ) - }) + continue + } // mark the (now stopped) record as Expired so it is not picked up again const [stoppedJob] = await this.db.getServiceJob(svc.serviceId, svc.owner) if (stoppedJob) { @@ -3671,9 +3682,35 @@ export class C2DEngineDocker extends C2DEngine { CORE_LOGGER.error(`Service ${job.serviceId} container died — ${reason}`) } + // Takes the per-service lifecycle lock or throws: a stop must not run while the loop's + // start pipeline or a restart owns the service — its by-name network removal would tear + // down the docker resources the other operation is creating (and vice versa). + private acquireServiceLifecycleLock(serviceId: string): void { + if (this.serviceOpsInFlight.has(serviceId)) { + throw new Error( + `Service ${serviceId} has a start/stop/restart operation in progress — retry shortly` + ) + } + this.serviceOpsInFlight.add(serviceId) + } + public override async stopService( serviceId: string, owner: string + ): Promise { + this.acquireServiceLifecycleLock(serviceId) + try { + return await this.doStopService(serviceId, owner) + } finally { + this.serviceOpsInFlight.delete(serviceId) + } + } + + // The actual stop. Must only run while holding the service lifecycle lock (see + // stopService above). + private async doStopService( + serviceId: string, + owner: string ): Promise { const [job] = await this.db.getServiceJob(serviceId, owner) if (!job) return null @@ -3728,6 +3765,33 @@ export class C2DEngineDocker extends C2DEngine { newUserData?: string, newDockerCmd?: string[], newDockerEntrypoint?: string[] + ): Promise { + // Lifecycle lock: without it the InternalLoop's orphan-recovery (which sees the + // intermediate PullImage/BuildImage status this method persists) tears down the + // network created here mid-restart → container.start() fails with + // "network ocean-svc- not found". + this.acquireServiceLifecycleLock(serviceId) + try { + return await this.doRestartService( + serviceId, + owner, + newUserData, + newDockerCmd, + newDockerEntrypoint + ) + } finally { + this.serviceOpsInFlight.delete(serviceId) + } + } + + // The actual restart. Must only run while holding the service lifecycle lock (see + // restartService above). + private async doRestartService( + serviceId: string, + owner: string, + newUserData?: string, + newDockerCmd?: string[], + newDockerEntrypoint?: string[] ): Promise { const [job] = await this.db.getServiceJob(serviceId, owner) if (!job) return null diff --git a/src/test/integration/services.test.ts b/src/test/integration/services.test.ts index aad98456e..bfc5ea305 100644 --- a/src/test/integration/services.test.ts +++ b/src/test/integration/services.test.ts @@ -782,6 +782,90 @@ describe('********** Service on Demand', () => { ) }) + it('(l3) SERVICE_RESTART survives InternalLoop ticks mid-restart (orphan-recovery race)', async function () { + this.timeout(DEFAULT_TEST_TIMEOUT * 4) + const engine: any = getDockerEngine() + const before = await getServiceJob(serviceId) + const oldContainerId = before.containerId + + // Delay the image pull so the job sits in PullImage across several InternalLoop + // ticks (cronTime = 2 s) — the exact window in which the loop's orphan-recovery + // used to tear down the network the restart had just created, failing the restart + // with "network ocean-svc- not found" (the live-node bug). + const originalPull = engine.pullImageRef + engine.pullImageRef = async (...args: any[]) => { + await sleep(5000) + return originalPull.apply(engine, args) + } + try { + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_RESTART) + const task: ServiceRestartCommand = { + command: PROTOCOL_COMMANDS.SERVICE_RESTART, + consumerAddress: addr, + nonce, + signature, + serviceId + } + const restartPromise = new ServiceRestartHandler(oceanNode).handle(task) + + // wait until the restart is actually mid-pull (deterministic, not sleep-based) + const deadline = Date.now() + 10_000 + while (Date.now() < deadline) { + const j = await getServiceJob(serviceId) + if (j?.status === ServiceStatusNumber.PullImage) break + await sleep(250) + } + + // concurrent lifecycle operations must be rejected while the restart holds the lock + const stopSig = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_STOP) + const stopResp = await new ServiceStopHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_STOP, + consumerAddress: stopSig.consumerAddress, + nonce: stopSig.nonce, + signature: stopSig.signature, + serviceId + } as ServiceStopCommand) + expect(stopResp.status.httpStatus).to.not.equal(200) + expect(String(stopResp.status.error)).to.contain('operation in progress') + + const retrySig = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_RESTART) + const retryResp = await new ServiceRestartHandler(oceanNode).handle({ + ...task, + nonce: retrySig.nonce, + signature: retrySig.signature + }) + expect(retryResp.status.httpStatus).to.not.equal(200) + expect(String(retryResp.status.error)).to.contain('operation in progress') + + // the original restart must complete unharmed by the loop ticks that fired mid-pull + const resp = await restartPromise + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + } finally { + engine.pullImageRef = originalPull + } + + // pollServiceStatus throws if the job lands in Error — which is exactly what the + // pre-fix orphan-recovery race produced ("Service start aborted (node restarted + // mid-start)" / "network ocean-svc- not found"). + const running = await pollServiceStatus(serviceId, ServiceStatusNumber.Running) + expect(running.containerId).to.not.equal(oldContainerId) + expect(running.endpoints[0].hostPort).to.equal(hostPort) + expect(running.expiresAt).to.equal(expiresAt) + + const res = await httpGetWithRetry(endpointUrl) + assert( + res.status === 200, + `expected nginx HTTP 200 after raced restart, got ${res.status}` + ) + }) + it('(m) SERVICE_STOP → Stopped, container + network removed', async function () { this.timeout(DEFAULT_TEST_TIMEOUT * 2) const before = await getServiceJob(serviceId) diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 5588c8eab..0c047a05c 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -1512,9 +1512,11 @@ describe('service start/restart Docker cleanup on failure', function () { // Bypass the Docker-specific constructor but keep the prototype methods. engine = Object.create(C2DEngineDocker.prototype) // Constructor field initializers don't run via Object.create, so seed the CPU - // pinning maps that releaseCpus/allocateCpus (called by cleanupServiceDocker) touch. + // pinning maps that releaseCpus/allocateCpus (called by cleanupServiceDocker) touch, + // and the per-service lifecycle lock taken by stopService/restartService. engine.cpuAllocations = new Map() engine.envCpuCoresMap = new Map() + engine.serviceOpsInFlight = new Set() // inspect is needed by removeServiceNetwork (cleanup resolves the network by // deterministic name/id and checks for attached containers before removing). network = { diff --git a/src/test/unit/service/serviceNetworkCleanup.test.ts b/src/test/unit/service/serviceNetworkCleanup.test.ts index e4eb5c064..28983d3d3 100644 --- a/src/test/unit/service/serviceNetworkCleanup.test.ts +++ b/src/test/unit/service/serviceNetworkCleanup.test.ts @@ -72,6 +72,7 @@ function makeEngine(): any { createNetwork: sinon.stub() } engine.cpuAllocations = new Map() + engine.serviceOpsInFlight = new Set() return engine } diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts new file mode 100644 index 000000000..3e9810b98 --- /dev/null +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -0,0 +1,237 @@ +import { expect, assert } from 'chai' +import sinon from 'sinon' +import { C2DEngineDocker } from '../../../components/c2d/compute_engine_docker.js' +import { ServiceStatusNumber, ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' + +const OWNER = '0x0000000000000000000000000000000000000001' +const SERVICE_ID = 'svc-race-1' +const CLUSTER_HASH = 'hash-1' + +// Flushes the microtask queue so an unawaited async call can progress through its +// (immediately-resolving) stubbed awaits up to the next genuinely pending promise. +function flush(): Promise { + return new Promise((resolve) => setImmediate(resolve)) +} + +function makeJob(overrides: Partial = {}): ServiceJob { + return { + serviceId: SERVICE_ID, + clusterHash: CLUSTER_HASH, + environment: 'env-1', + owner: OWNER, + image: 'img', + tag: 'latest', + containerImage: 'img:latest', + containerId: 'c1', + networkId: '', + status: ServiceStatusNumber.Running, + statusText: 'Running', + dateCreated: new Date(0).toISOString(), + expiresAt: Date.now() + 3600_000, + duration: 3600, + exposedPorts: [8888], + endpoints: [{ containerPort: 8888, hostPort: 31000, url: 'http://localhost:31000' }], + userData: undefined, // decryptUserData(undefined) → {} — keyManager never touched + resources: [], // no cpu request → allocateCpus is skipped + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '0xl', + claimTx: '0xc', + cancelTx: '', + cost: 5 + }, + ...overrides + } +} + +function benignError(statusCode: number = 404) { + const e: any = new Error(`docker ${statusCode}`) + e.statusCode = statusCode + return e +} + +// Bypass the Docker-specific constructor while retaining the prototype chain (same +// pattern as serviceNetworkCleanup.test.ts) so private methods/fields can be exercised +// with a stubbed docker + db. +function makeEngine(): any { + const engine: any = Object.create(C2DEngineDocker.prototype) + engine.docker = { + getNetwork: sinon.stub(), + getContainer: sinon.stub(), + createNetwork: sinon.stub(), + createContainer: sinon.stub() + } + // networks are "already gone" by default (benign 404 on inspect) + engine.docker.getNetwork.returns({ + inspect: sinon.stub().rejects(benignError(404)), + remove: sinon.stub().resolves(undefined) + }) + engine.db = { + getServiceJob: sinon.stub().resolves([]), + updateServiceJob: sinon.stub().resolves(1), + getRunningJobs: sinon.stub().resolves([]), + getPendingServiceStarts: sinon.stub().resolves([]), + getExpiredServiceJobs: sinon.stub().resolves([]) + } + engine.cpuAllocations = new Map() + engine.serviceOpsInFlight = new Set() + engine.serviceStartPromises = new Set() + engine.clusterConfig = { + hash: CLUSTER_HASH, + connection: { resources: [], serviceOnDemand: {} } + } + engine.stopped = false + engine.isInternalLoopRunning = false + engine.cronTimer = null + engine.setNewTimer = sinon.stub() + engine.checkRunningServices = sinon.stub().resolves() + return engine +} + +function stubContainer(overrides: Record = {}) { + return { + stop: sinon.stub().resolves(undefined), + remove: sinon.stub().resolves(undefined), + start: sinon.stub().resolves(undefined), + ...overrides + } +} + +describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { + afterEach(() => sinon.restore()) + + it('restartService holds the lock while the image pull is pending and releases it after', async () => { + const engine = makeEngine() + const job = makeJob() + engine.db.getServiceJob.resolves([job]) + engine.docker.getContainer.returns(stubContainer()) + engine.docker.createNetwork.resolves({ id: 'newnet' }) + const newContainer = stubContainer() + ;(newContainer as any).id = 'newc' + engine.docker.createContainer.resolves(newContainer) + let resolvePull: () => void + engine.pullImageRef = () => + new Promise((resolve) => { + resolvePull = resolve + }) + + const restart = engine.restartService(SERVICE_ID, OWNER) + await flush() + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(true) + + resolvePull!() + const restarted = await restart + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(restarted.status).to.equal(ServiceStatusNumber.Running) + expect(restarted.containerId).to.equal('newc') + }) + + it('restartService releases the lock on the failure path too', async () => { + const engine = makeEngine() + engine.db.getServiceJob.resolves([makeJob()]) + engine.docker.getContainer.returns(stubContainer()) + engine.pullImageRef = sinon.stub().rejects(new Error('pull failed')) + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.equal('pull failed') + } + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + }) + + it('rejects a concurrent restart/stop while the lock is held, without touching docker or the DB', async () => { + const engine = makeEngine() + engine.serviceOpsInFlight.add(SERVICE_ID) + + for (const call of [ + () => engine.restartService(SERVICE_ID, OWNER), + () => engine.stopService(SERVICE_ID, OWNER) + ]) { + try { + await call() + expect.fail('expected the call to reject while the lock is held') + } catch (e: any) { + expect(e.message).to.contain('operation in progress') + } + } + expect(engine.docker.getContainer.called).to.equal(false) + expect(engine.docker.getNetwork.called).to.equal(false) + expect(engine.db.getServiceJob.called).to.equal(false) + expect(engine.db.updateServiceJob.called).to.equal(false) + // a failed acquire must not clear a lock held by the in-flight operation + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(true) + }) + + it('stopService holds the lock while stopping (a mid-stop restart rejects) and releases it after', async () => { + const engine = makeEngine() + const job = makeJob() + engine.db.getServiceJob.resolves([job]) + let resolveStop: () => void + const pendingStop = new Promise((resolve) => { + resolveStop = resolve + }) + engine.docker.getContainer.returns(stubContainer({ stop: () => pendingStop })) + + const stop = engine.stopService(SERVICE_ID, OWNER) + await flush() + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(true) + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject while a stop is in flight') + } catch (e: any) { + expect(e.message).to.contain('operation in progress') + } + + resolveStop!() + const stopped = await stop + expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + }) + + it('InternalLoop does not launch processServiceStart for a locked service (the orphan-recovery race)', async () => { + const engine = makeEngine() + const midRestart = makeJob({ + status: ServiceStatusNumber.PullImage, + statusText: 'PullImage' + }) + engine.db.getPendingServiceStarts.resolves([midRestart]) + engine.processServiceStart = sinon.stub().resolves() + + // serviceId locked (restartService in flight) → the loop must skip it + engine.serviceOpsInFlight.add(SERVICE_ID) + await engine.InternalLoop() + expect(engine.processServiceStart.called).to.equal(false) + + // control: with the lock free the loop DOES pick the job up (orphan recovery), + // proving this test fails without the lock + engine.serviceOpsInFlight.clear() + engine.isInternalLoopRunning = false + await engine.InternalLoop() + await flush() + expect(engine.processServiceStart.calledOnceWith(midRestart)).to.equal(true) + }) + + it('the expiry sweep defers a locked service instead of marking it Expired without teardown', async () => { + const engine = makeEngine() + const expired = makeJob({ expiresAt: Date.now() - 1000 }) + engine.db.getExpiredServiceJobs.resolves([expired]) + engine.db.getServiceJob.resolves([expired]) + engine.docker.getContainer.returns(stubContainer()) + + // locked (e.g. a user restart is mid-pull) → nothing stopped, nothing persisted + engine.serviceOpsInFlight.add(SERVICE_ID) + await engine.InternalLoop() + expect(engine.db.updateServiceJob.called).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + assert(expired.status === ServiceStatusNumber.Running, 'status must be untouched') + + // lock free → the sweep stops the service and marks it Expired + engine.serviceOpsInFlight.clear() + engine.isInternalLoopRunning = false + await engine.InternalLoop() + expect(expired.status).to.equal(ServiceStatusNumber.Expired) + }) +}) From 828950845855d568a80fa06f6546c96af9f4fba2 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Wed, 15 Jul 2026 09:33:28 +0300 Subject: [PATCH 02/10] fix reviews --- src/components/c2d/compute_engine_docker.ts | 67 ++++++++++++------- src/test/unit/compute.test.ts | 1 + .../service/serviceNetworkCleanup.test.ts | 1 + .../unit/service/serviceRestartRace.test.ts | 29 +++++++- 4 files changed, 71 insertions(+), 27 deletions(-) diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index b1f3857a3..73d69432e 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -112,10 +112,11 @@ export class C2DEngineDocker extends C2DEngine { // "network ocean-svc- not found" failure) and start/stop/restart clobber each // other's docker resources and job status. private serviceOpsInFlight: Set = new Set() - // The in-flight processServiceStart() promises (launched fire-and-forget by InternalLoop), - // so stop() can drain them before returning — otherwise a start could outlive stop() and - // race a restarted engine on the same shared DB. - private serviceStartPromises: Set> = new Set() + // The in-flight service lifecycle promises — processServiceStart() launched + // fire-and-forget by InternalLoop, plus handler-driven stopService()/restartService() + // calls — so stop() can drain them before returning. Otherwise an op could outlive + // stop() and race a restarted engine on the same shared DB. + private serviceOpPromises: Set> = new Set() private imageCleanupTimer: NodeJS.Timeout | null = null private paymentClaimTimer: NodeJS.Timeout | null = null private scanDBUpdateTimer: NodeJS.Timeout | null = null @@ -661,11 +662,12 @@ export class C2DEngineDocker extends C2DEngine { await this.internalLoopPromise.catch(() => {}) this.internalLoopPromise = null } - // Drain any in-flight service-start pipelines launched by the loop, so none continue - // (and touch escrow/Docker on the shared DB) after the engine is considered stopped. - if (this.serviceStartPromises.size > 0) { - await Promise.allSettled([...this.serviceStartPromises]) - this.serviceStartPromises.clear() + // Drain any in-flight service lifecycle ops — start pipelines launched by the loop + // AND handler-driven stopService/restartService calls — so none continue (and touch + // escrow/Docker on the shared DB) after the engine is considered stopped. + if (this.serviceOpPromises.size > 0) { + await Promise.allSettled([...this.serviceOpPromises]) + this.serviceOpPromises.clear() } this.isInternalLoopRunning = false // Stop image cleanup timer @@ -1797,9 +1799,9 @@ export class C2DEngineDocker extends C2DEngine { // Track the promise so stop() can drain it; clean both trackers when it settles. const startPromise = this.processServiceStart(svc).finally(() => { this.serviceOpsInFlight.delete(svc.serviceId) - this.serviceStartPromises.delete(startPromise) + this.serviceOpPromises.delete(startPromise) }) - this.serviceStartPromises.add(startPromise) + this.serviceOpPromises.add(startPromise) } // Service-on-Demand expiry: stop services whose paid window has elapsed. @@ -3641,8 +3643,12 @@ export class C2DEngineDocker extends C2DEngine { // detected within ~cronTime instead of only at expiresAt. private async checkRunningServices(): Promise { const services = await this.db.getRunningServiceJobs(this.getC2DConfig().hash) + // Skip services with a lifecycle op in flight: a restart intentionally kills the + // container mid-way, which must not be reported as an unexpected death. const runningOnly = services.filter( - (svc) => svc.status === ServiceStatusNumber.Running + (svc) => + svc.status === ServiceStatusNumber.Running && + !this.serviceOpsInFlight.has(svc.serviceId) ) await Promise.all(runningOnly.map((svc) => this.checkServiceContainerHealth(svc))) } @@ -3676,6 +3682,11 @@ export class C2DEngineDocker extends C2DEngine { // already moved on (stopped/restarted/expired) by another path — don't clobber it return } + if (fresh.containerId !== job.containerId) { + // the container we observed dead is no longer the job's container (a restart + // replaced it since our snapshot) — the current one is not known to be dead + return + } fresh.status = ServiceStatusNumber.Error fresh.statusText = `service container exited unexpectedly: ${reason}` await this.db.updateServiceJob(fresh) @@ -3699,11 +3710,13 @@ export class C2DEngineDocker extends C2DEngine { owner: string ): Promise { this.acquireServiceLifecycleLock(serviceId) - try { - return await this.doStopService(serviceId, owner) - } finally { + // Tracked like loop-launched starts so engine stop() drains an in-flight stop too. + const op = this.doStopService(serviceId, owner).finally(() => { this.serviceOpsInFlight.delete(serviceId) - } + this.serviceOpPromises.delete(op) + }) + this.serviceOpPromises.add(op) + return await op } // The actual stop. Must only run while holding the service lifecycle lock (see @@ -3771,17 +3784,19 @@ export class C2DEngineDocker extends C2DEngine { // network created here mid-restart → container.start() fails with // "network ocean-svc- not found". this.acquireServiceLifecycleLock(serviceId) - try { - return await this.doRestartService( - serviceId, - owner, - newUserData, - newDockerCmd, - newDockerEntrypoint - ) - } finally { + // Tracked like loop-launched starts so engine stop() drains an in-flight restart too. + const op = this.doRestartService( + serviceId, + owner, + newUserData, + newDockerCmd, + newDockerEntrypoint + ).finally(() => { this.serviceOpsInFlight.delete(serviceId) - } + this.serviceOpPromises.delete(op) + }) + this.serviceOpPromises.add(op) + return await op } // The actual restart. Must only run while holding the service lifecycle lock (see diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 0c047a05c..8ff5a4651 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -1517,6 +1517,7 @@ describe('service start/restart Docker cleanup on failure', function () { engine.cpuAllocations = new Map() engine.envCpuCoresMap = new Map() engine.serviceOpsInFlight = new Set() + engine.serviceOpPromises = new Set() // inspect is needed by removeServiceNetwork (cleanup resolves the network by // deterministic name/id and checks for attached containers before removing). network = { diff --git a/src/test/unit/service/serviceNetworkCleanup.test.ts b/src/test/unit/service/serviceNetworkCleanup.test.ts index 28983d3d3..70e4d15cb 100644 --- a/src/test/unit/service/serviceNetworkCleanup.test.ts +++ b/src/test/unit/service/serviceNetworkCleanup.test.ts @@ -73,6 +73,7 @@ function makeEngine(): any { } engine.cpuAllocations = new Map() engine.serviceOpsInFlight = new Set() + engine.serviceOpPromises = new Set() return engine } diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts index 3e9810b98..0ff6a43da 100644 --- a/src/test/unit/service/serviceRestartRace.test.ts +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -76,7 +76,7 @@ function makeEngine(): any { } engine.cpuAllocations = new Map() engine.serviceOpsInFlight = new Set() - engine.serviceStartPromises = new Set() + engine.serviceOpPromises = new Set() engine.clusterConfig = { hash: CLUSTER_HASH, connection: { resources: [], serviceOnDemand: {} } @@ -214,6 +214,33 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { expect(engine.processServiceStart.calledOnceWith(midRestart)).to.equal(true) }) + it('engine stop() drains an in-flight handler-driven stopService before returning', async () => { + const engine = makeEngine() + engine.db.getServiceJob.resolves([makeJob()]) + let resolveStop: () => void + const pendingStop = new Promise((resolve) => { + resolveStop = resolve + }) + engine.docker.getContainer.returns(stubContainer({ stop: () => pendingStop })) + + const serviceStop = engine.stopService(SERVICE_ID, OWNER) + await flush() + expect(engine.serviceOpPromises.size).to.equal(1) + + let drained = false + const engineStop = engine.stop().then(() => { + drained = true + }) + await flush() + expect(drained).to.equal(false) // must wait on the in-flight stopService + + resolveStop!() + await engineStop + expect(drained).to.equal(true) + expect((await serviceStop).status).to.equal(ServiceStatusNumber.Stopped) + expect(engine.serviceOpPromises.size).to.equal(0) + }) + it('the expiry sweep defers a locked service instead of marking it Expired without teardown', async () => { const engine = makeEngine() const expired = makeJob({ expiresAt: Date.now() - 1000 }) From 42eeeb3f0ab4ed538ecbd2cf2cd132885af52d10 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Wed, 15 Jul 2026 09:46:38 +0300 Subject: [PATCH 03/10] add comment --- src/components/c2d/compute_engine_docker.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 73d69432e..b4fc218ab 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -3816,7 +3816,12 @@ export class C2DEngineDocker extends C2DEngine { if (job.status === ServiceStatusNumber.Expired || Date.now() >= job.expiresAt) throw new Error('Cannot restart an expired service') - // 1. Tear down existing container + network (best-effort) + // 1. Tear down existing container + network (best-effort). Deliberately BEFORE + // persisting Starting: a bare Starting record (cleared ids) is treated as a brand-new + // start by the boot-time loop, which would create+claim a SECOND escrow lock and + // allocate NEW host ports. Crashing here instead leaves the job Running with stale + // ids — benign: the boot health check flips it to Error and a re-issued restart + // recovers on the same ports/payment (teardown 404s are swallowed). if (job.containerId) { const c = this.docker.getContainer(job.containerId) await c.stop({ t: 10 }).catch(() => {}) From 50af2123ce228620e0d97b4ca3570472d0f57e9c Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 16 Jul 2026 21:18:24 +0300 Subject: [PATCH 04/10] Service lifecycle hardening --- docs/services.md | 36 +- src/@types/C2D/ServiceOnDemand.ts | 16 + src/components/c2d/compute_engine_docker.ts | 423 +++++++++++++++--- src/components/core/service/extendService.ts | 31 +- .../core/service/getStreamableLogs.ts | 31 +- src/components/core/service/restartService.ts | 36 +- src/components/core/service/stopService.ts | 31 +- src/components/core/service/utils.ts | 28 ++ src/components/database/C2DDatabase.ts | 23 + src/components/database/sqliteCompute.ts | 149 +++++- src/test/integration/services.test.ts | 7 +- src/test/unit/compute.test.ts | 46 +- src/test/unit/service/serviceHandlers.test.ts | 8 +- .../unit/service/serviceJobsDatabase.test.ts | 80 +++- .../unit/service/serviceRestartRace.test.ts | 225 +++++++++- 15 files changed, 993 insertions(+), 177 deletions(-) diff --git a/docs/services.md b/docs/services.md index 6e2d49d7f..c7b3d0d42 100644 --- a/docs/services.md +++ b/docs/services.md @@ -29,8 +29,8 @@ and `signature` as query parameters (or an auth-token `Authorization` header). | `SERVICE_START` | `/api/services/serviceStart` | POST | Validate, persist a `Starting` record, and return the `serviceId` immediately (escrow + image + container happen in the background) | | `SERVICE_GET_STATUS` | `/api/services/serviceStatus` | GET | Read job status / endpoints — authenticated, owner-scoped (see notice below); poll this to follow a starting service | | `SERVICE_EXTEND` | `/api/services/serviceExtend` | POST | Pay to push the expiry further out | -| `SERVICE_RESTART` | `/api/services/serviceRestart` | POST | Recreate the container (no extra charge) | -| `SERVICE_STOP` | `/api/services/serviceStop` | POST | Tear down the container and release resources | +| `SERVICE_RESTART` | `/api/services/serviceRestart` | POST | Recreate the container (no extra charge); asynchronous like start — returns once the job is `Restarting`, poll `serviceStatus` | +| `SERVICE_STOP` | `/api/services/serviceStop` | POST | Tear down the container; the paid resource reservation (cpu/ram/gpu + host ports) is kept until `expiresAt`, so the service can be restarted anytime on the same endpoints | | `SERVICE_GET_TEMPLATES` | `/api/services/serviceTemplates` | GET | List operator-published service templates | | `SERVICE_GET_STREAMABLE_LOGS` | `/api/services/serviceStreamableLogs` | GET | Stream the container's live stdout/stderr logs — authenticated, owner-scoped; available while `Running` or `Error`; optional `since` to skip history | @@ -57,16 +57,33 @@ container creation fails before the claim, the lock is **cancelled (refunded)** in a `*Failed` / `Error` status. This is a change from the previous synchronous flow, which locked-then-claimed up front. +**Restart is asynchronous too.** `serviceRestart` performs only the fast validations +(ownership, environment/access, not expired, payment not refunded), persists the job as +`Restarting (45)` and responds immediately — the teardown, image re-pull/rebuild and new +container happen in the background under the same per-service lock. Poll `serviceStatus` +and watch `Restarting` → `PullImage`/`BuildImage` → `Running` (or `Error` with the failure +reason in `statusText`). A service whose start payment was **refunded** (lock cancelled +before it was claimed) cannot be restarted — it was never paid for; start a new service. + +**The reservation lasts the whole paid window — only `Expired` releases it.** The consumer +paid for the resources for a time interval and may use them as they please within it: +running the service, stopping it, restarting it. An explicit `SERVICE_STOP` therefore tears +down the container/network but **keeps** the resource amounts (cpu/ram/gpu) counted and the +host ports reserved — another consumer cannot take them, and a restart resumes on the same +endpoints. Once `expiresAt` passes, the expiry sweep tears down whatever is left, marks the +job `Expired`, and only then releases everything. The sweep refuses to mark `Expired` while +teardown fails (e.g. Docker unreachable) — the job stays `Error` and is retried every tick, +so a resource release is never silently skipped. + **`Running` is monitored too.** The same background loop that advances a starting service also checks every `Running` service's container on each tick (~every few seconds). If the container exits on its own — crash, OOM, or the Docker daemon itself becoming unreachable — the job is moved to `Error` immediately instead of waiting for `expiresAt`. This health check does **not** release the service's reserved host ports/network/container record, since the consumer already paid for them; use `SERVICE_RESTART` to bring the service back on the same endpoints. `Error` -counts as an active/resource-reserving status just like `Running` does — it still occupies its -cpu/ram/gpu allocation and keeps its host ports held — until it is restarted, explicitly -stopped, or swept by the same expiry check once `expiresAt` passes (which then fully releases -everything, same as a normal expiry). +counts as an active/resource-reserving status just like `Running` and `Stopped` do — it still +occupies its cpu/ram/gpu allocation and keeps its host ports held — until it is restarted or +swept by the expiry check once `expiresAt` passes (which then fully releases everything). **Restart is self-healing with respect to leftover Docker state.** Each service gets a Docker network with the deterministic name `ocean-svc-`. Teardown (restart, stop, expiry @@ -95,6 +112,13 @@ restart that had just created it, failing the restart with `network ocean-svc- not found`. If a service expires while such an operation is in flight, the expiry sweep simply retries on a later tick. +Exclusivity holds **across node processes** too, not just within one: each operation also +takes a lease row in the SQLite `service_locks` table, so two processes sharing the same +`databases/` directory and Docker daemon (e.g. an old container still running during a +redeploy) cannot run conflicting operations on the same service. Leases are heartbeated +every 30 s while the operation runs; a lease not refreshed for 2 minutes belongs to a +crashed process and is stolen automatically, so no manual cleanup is ever needed. + ## Configuration Service-on-demand is configured per Docker connection under `serviceOnDemand`: diff --git a/src/@types/C2D/ServiceOnDemand.ts b/src/@types/C2D/ServiceOnDemand.ts index 8693022a4..f692862b1 100644 --- a/src/@types/C2D/ServiceOnDemand.ts +++ b/src/@types/C2D/ServiceOnDemand.ts @@ -78,6 +78,7 @@ export enum ServiceStatusNumber { Locking = 20, // escrow createLock in progress (funds locked, not yet claimed) Claiming = 30, // payment phase: claimLock on success, or cancelLock if the image step failed Running = 40, + Restarting = 45, // SERVICE_RESTART accepted; teardown + re-pull/build + new container in progress Stopping = 50, Stopped = 70, Expired = 75, @@ -95,12 +96,27 @@ export const ServiceStatusText: Record = { [ServiceStatusNumber.Locking]: 'Locking', [ServiceStatusNumber.Claiming]: 'Claiming', [ServiceStatusNumber.Running]: 'Running', + [ServiceStatusNumber.Restarting]: 'Restarting', [ServiceStatusNumber.Stopping]: 'Stopping', [ServiceStatusNumber.Stopped]: 'Stopped', [ServiceStatusNumber.Expired]: 'Expired', [ServiceStatusNumber.Error]: 'Error' } +// Statuses of a service job that is mid-start/restart and owned by an exclusive +// lifecycle operation. Single source of truth for getPendingServiceStarts (DB query) and +// the pipeline's staleness guard — the two MUST agree or a job can be picked up and then +// ignored (or vice versa). Restarting is included so a job orphaned by a crash +// mid-restart is recovered at boot exactly like a crash mid-start. +export const SERVICE_START_PENDING_STATUSES: readonly ServiceStatusNumber[] = [ + ServiceStatusNumber.Starting, + ServiceStatusNumber.Locking, + ServiceStatusNumber.PullImage, + ServiceStatusNumber.BuildImage, + ServiceStatusNumber.Claiming, + ServiceStatusNumber.Restarting +] + export interface ServiceJob { serviceId: string // unique id for a running service — distinct from a compute jobId clusterHash: string diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index b4fc218ab..3698191f9 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -68,13 +68,15 @@ import { import { getAddress, ZeroAddress } from 'ethers' import { ServiceStatusNumber, - ServiceStatusText + ServiceStatusText, + SERVICE_START_PENDING_STATUSES } from '../../@types/C2D/ServiceOnDemand.js' import type { ServiceJob } from '../../@types/C2D/ServiceOnDemand.js' import { resolveServiceImage } from './serviceResourceMatching.js' import { allocateHostPort, releaseHostPort, + reserveHostPort, seedAllocatedPorts, userDataToEnv, decryptUserData @@ -85,6 +87,19 @@ const C2D_CONTAINER_GID = 1000 const trivyImage = 'aquasec/trivy:0.69.3' // Use pinned versions for safety +// Cross-process service lifecycle lock timing: locks are heartbeated every +// SERVICE_LOCK_HEARTBEAT_MS while an operation runs, and a lock not refreshed within +// SERVICE_LOCK_STALE_MS belongs to a crashed process and may be stolen. Staleness must +// be a comfortable multiple of the heartbeat so one missed beat (GC pause, busy DB) +// doesn't get a live operation's resources torn down under it. +const SERVICE_LOCK_HEARTBEAT_MS = 30_000 +const SERVICE_LOCK_STALE_MS = 120_000 + +// Identifies one engine instance as a service-lock holder across processes. pid alone is +// not enough: pids are reused, and one process can host several engine instances. +const makeServiceLockHolderId = () => + `${process.pid}:${Math.random().toString(36).slice(2, 10)}` + // "Already gone" Docker errors (404 missing, 304 already stopped) are the desired end // state of a teardown, so treat them as success. const isBenignDockerError = (e: any) => e?.statusCode === 404 || e?.statusCode === 304 @@ -110,8 +125,12 @@ export class C2DEngineDocker extends C2DEngine { // (including its expiry sweep), restart/stop throw. Without this, the loop's // orphan-recovery tears down the network a concurrent restart just created (the // "network ocean-svc- not found" failure) and start/stop/restart clobber each - // other's docker resources and job status. + // other's docker resources and job status. The in-memory set serializes within this + // process; the service_locks DB lease (acquired alongside it) extends the guarantee + // across processes sharing the same DB file + Docker daemon. private serviceOpsInFlight: Set = new Set() + private readonly serviceLockHolderId: string = makeServiceLockHolderId() + private serviceLockHeartbeatTimer: NodeJS.Timeout | null = null // The in-flight service lifecycle promises — processServiceStart() launched // fire-and-forget by InternalLoop, plus handler-driven stopService()/restartService() // calls — so stop() can drain them before returning. Otherwise an op could outlive @@ -572,6 +591,17 @@ export class C2DEngineDocker extends C2DEngine { return } + // Heartbeat the DB rows of every service lifecycle lock this instance holds, so a + // multi-minute image pull/build isn't stolen as a stale lock by another process. + if (!this.serviceLockHeartbeatTimer) { + this.serviceLockHeartbeatTimer = setInterval(() => { + if (this.serviceOpsInFlight.size === 0) return + this.db.refreshServiceLocks(this.serviceLockHolderId).catch((e) => { + CORE_LOGGER.warn(`service lock heartbeat failed: ${e.message}`) + }) + }, SERVICE_LOCK_HEARTBEAT_MS) + } + // Start image cleanup timer if (this.cleanupInterval) { if (this.imageCleanupTimer) { @@ -681,6 +711,10 @@ export class C2DEngineDocker extends C2DEngine { this.paymentClaimTimer = null CORE_LOGGER.debug('Payment claim timer stopped') } + if (this.serviceLockHeartbeatTimer) { + clearInterval(this.serviceLockHeartbeatTimer) + this.serviceLockHeartbeatTimer = null + } } public async updateImageUsage(image: string): Promise { @@ -1794,12 +1828,12 @@ export class C2DEngineDocker extends C2DEngine { this.getC2DConfig().hash ) for (const svc of pendingStarts) { - if (this.serviceOpsInFlight.has(svc.serviceId)) continue - this.serviceOpsInFlight.add(svc.serviceId) + // eslint-disable-next-line no-await-in-loop + if (!(await this.tryAcquireServiceLifecycleLock(svc.serviceId))) continue // Track the promise so stop() can drain it; clean both trackers when it settles. const startPromise = this.processServiceStart(svc).finally(() => { - this.serviceOpsInFlight.delete(svc.serviceId) this.serviceOpPromises.delete(startPromise) + return this.releaseServiceLifecycleLock(svc.serviceId) }) this.serviceOpPromises.add(startPromise) } @@ -1810,8 +1844,9 @@ export class C2DEngineDocker extends C2DEngine { ) for (const svc of expiredServices) { CORE_LOGGER.info(`Service ${svc.serviceId} expired — stopping`) + let stopped: ServiceJob | null try { - await this.stopService(svc.serviceId, svc.owner) + stopped = await this.stopService(svc.serviceId, svc.owner) } catch (e: any) { // Typically the lifecycle lock (a restart/stop already in flight). Marking // Expired without teardown would leak the container/ports, so defer the whole @@ -1821,12 +1856,37 @@ export class C2DEngineDocker extends C2DEngine { ) continue } + // Flip to Expired ONLY once teardown actually completed. Expired is terminal — + // it is never swept again — so marking it while the stop failed mid-way (the + // job comes back as Error "stop failed: …" with its container possibly still + // running and its ports still reserved) would leak those resources forever. + // Left as Error, the job stays in the expirable set and is retried next tick. + if ( + !stopped || + (stopped.status !== ServiceStatusNumber.Stopped && + stopped.status !== ServiceStatusNumber.Expired) + ) { + CORE_LOGGER.error( + `Expired service ${svc.serviceId} teardown did not complete ` + + `(status "${stopped?.statusText ?? 'gone'}") — retrying next tick` + ) + continue + } // mark the (now stopped) record as Expired so it is not picked up again const [stoppedJob] = await this.db.getServiceJob(svc.serviceId, svc.owner) if (stoppedJob) { stoppedJob.status = ServiceStatusNumber.Expired stoppedJob.statusText = ServiceStatusText[ServiceStatusNumber.Expired] await this.db.updateServiceJob(stoppedJob) + // Expired is the ONLY transition that releases the reservation: amounts stop + // counting (Expired is not an active status) and the host ports are freed + // here. Everywhere else — including an explicit user stop — the paid-for + // reservation survives so the service can be restarted on the same endpoints. + for (const ep of stoppedJob.endpoints ?? []) releaseHostPort(ep.hostPort) + CORE_LOGGER.debug( + `Service ${svc.serviceId} marked Expired — all resources released ` + + `(ports [${(stoppedJob.endpoints ?? []).map((ep) => ep.hostPort).join(',')}])` + ) } } } catch (e) { @@ -3328,17 +3388,43 @@ export class C2DEngineDocker extends C2DEngine { // Claiming → Running. Escrow is reordered vs. the old sync flow: createLock first, claimLock // only AFTER the image succeeds, cancelLock (refund) if the image step fails. Never throws — // every terminal outcome is persisted as the job status so clients see it via serviceStatus. - public override async processServiceStart(job: ServiceJob): Promise { + public override async processServiceStart(snapshot: ServiceJob): Promise { + // Re-read the job under the lifecycle lock: the loop's pendingStarts snapshot was + // queried BEFORE the lock was acquired, so it can be stale — e.g. captured while a + // restart was mid-pull and processed after that restart completed and released the + // lock. Acting on the stale row would tear down the freshly created container + + // network (and redo escrow ops) of a service that is actually fine. + const [job] = await this.db.getServiceJob(snapshot.serviceId, snapshot.owner) + if (!job || !SERVICE_START_PENDING_STATUSES.includes(job.status)) { + CORE_LOGGER.debug( + `processServiceStart ${snapshot.serviceId}: stale snapshot ` + + `(snapshot status "${snapshot.statusText}", fresh status ` + + `"${job?.statusText ?? 'row gone'}") — nothing to do` + ) + return + } + const { serviceId } = job const { chainId, token } = job.payment + CORE_LOGGER.debug( + `processServiceStart ${serviceId}: picked up in status "${job.statusText}" ` + + `(image=${job.containerImage}, owner=${job.owner})` + ) - // Orphan recovery: a previous process died mid-start (after a restart the in-memory loop - // guard is empty, so an intermediate-state record reaches here). Refund any unclaimed lock, - // tear down partial docker, release ports, and mark Error — never resume on-chain ops. + // Orphan recovery: a previous process died mid-start/mid-restart (after a node + // restart the in-memory loop guard is empty, so an intermediate-state record reaches + // here). Refund any unclaimed lock, tear down partial docker, and mark Error — never + // resume on-chain ops. A PAID (claimed) job keeps its host ports reserved: it stays + // restartable on the same endpoints until expiresAt (the expiry sweep releases them, + // and seedAllocatedPorts re-seeds them at boot). An unpaid one frees them. if (job.status !== ServiceStatusNumber.Starting) { CORE_LOGGER.error( `processServiceStart: orphaned service ${serviceId} in state "${job.statusText}" — cleaning up` ) + await this.logServiceDockerState( + `orphan recovery ${serviceId} (state "${job.statusText}")`, + serviceId + ) if (job.payment.lockTx && !job.payment.claimTx && !job.payment.cancelTx) { job.payment.cancelTx = await this.safeCancelLock( chainId, @@ -3353,8 +3439,12 @@ export class C2DEngineDocker extends C2DEngine { null, serviceId ) - await this.removeServiceNetwork(serviceId, job.networkId).catch(() => {}) - for (const ep of job.endpoints) releaseHostPort(ep.hostPort) + await this.removeServiceNetwork(serviceId, job.networkId).catch((e) => { + CORE_LOGGER.debug(`orphan recovery ${serviceId}: network removal: ${e.message}`) + }) + if (!job.payment.claimTx) { + for (const ep of job.endpoints) releaseHostPort(ep.hostPort) + } job.status = ServiceStatusNumber.Error job.statusText = 'Service start aborted (node restarted mid-start)' await this.db.updateServiceJob(job) @@ -3520,6 +3610,9 @@ export class C2DEngineDocker extends C2DEngine { PidsLimit: 512 } }) + CORE_LOGGER.debug( + `start ${serviceId}: created container ${container.id} on network ${network.id} — starting` + ) await container.start() job.containerId = container.id @@ -3527,9 +3620,16 @@ export class C2DEngineDocker extends C2DEngine { job.status = ServiceStatusNumber.Running job.statusText = ServiceStatusText[ServiceStatusNumber.Running] await this.db.updateServiceJob(job) + CORE_LOGGER.info( + `start ${serviceId}: completed — container ${container.id} Running on ` + + `ports [${job.endpoints.map((ep) => ep.hostPort).join(',')}]` + ) } catch (err: any) { + await this.logServiceDockerState( + `start ${serviceId}: FAILED (${err.message}) — docker state at failure`, + serviceId + ) await this.cleanupServiceDocker(container, network, serviceId) - for (const ep of job.endpoints) releaseHostPort(ep.hostPort) // Refund if funds were locked but never claimed (e.g. container creation failed). if (job.payment.lockTx && !job.payment.claimTx && !job.payment.cancelTx) { job.payment.cancelTx = await this.safeCancelLock( @@ -3539,6 +3639,13 @@ export class C2DEngineDocker extends C2DEngine { job.owner ) } + if (!job.payment.claimTx) { + // Never paid (lock refunded): the job is not restartable, so free its ports. + for (const ep of job.endpoints) releaseHostPort(ep.hostPort) + } + // Paid (claimed) failures keep their host ports reserved: the consumer paid until + // expiresAt and may SERVICE_RESTART on the same endpoints anytime — the expiry + // sweep releases them once the window elapses. job.status = ServiceStatusNumber.Error job.statusText = String(err.message) await this.db.updateServiceJob(job) @@ -3562,6 +3669,37 @@ export class C2DEngineDocker extends C2DEngine { } } + // DEBUG-level dump of the Docker daemon state relevant to services: every container + // (docker ps -a, compacted) and every ocean-svc-* network. Pure diagnostics so a + // lifecycle failure on a remote node can be reconstructed from its logs — never throws. + private async logServiceDockerState(context: string, serviceId?: string) { + try { + const [containers, networks] = await Promise.all([ + this.docker.listContainers({ all: true }), + this.docker.listNetworks() + ]) + const cs = containers.map((c: any) => ({ + id: c.Id?.slice(0, 12), + names: c.Names, + state: c.State, + status: c.Status, + networks: Object.keys(c.NetworkSettings?.Networks ?? {}) + })) + const ns = networks + .filter( + (n: any) => + n.Name?.startsWith('ocean-svc-') || (serviceId && n.Name?.includes(serviceId)) + ) + .map((n: any) => ({ id: n.Id?.slice(0, 12), name: n.Name })) + CORE_LOGGER.debug( + `[docker state] ${context}: ocean-svc networks=${JSON.stringify(ns)} ` + + `containers=${JSON.stringify(cs)}` + ) + } catch (e: any) { + CORE_LOGGER.debug(`[docker state] ${context}: unavailable (${e.message})`) + } + } + // Best-effort teardown of a half-created service container + network. Used by the // startService / restartService failure paths to avoid leaking Docker networks // (which would exhaust the daemon's IPAM CIDR pool over repeated failures). The network @@ -3601,10 +3739,18 @@ export class C2DEngineDocker extends C2DEngine { // eslint-disable-next-line no-await-in-loop info = await network.inspect() } catch (e: any) { - if (isBenignDockerError(e)) continue + if (isBenignDockerError(e)) { + CORE_LOGGER.debug(`removeServiceNetwork ${serviceId}: "${ref}" already gone`) + continue + } throw e } - for (const containerId of Object.keys(info?.Containers ?? {})) { + const attached = Object.keys(info?.Containers ?? {}) + CORE_LOGGER.debug( + `removeServiceNetwork ${serviceId}: removing "${ref}" (id=${info?.Id?.slice(0, 12)}, ` + + `attached containers=[${attached.map((c) => c.slice(0, 12)).join(',')}])` + ) + for (const containerId of attached) { // eslint-disable-next-line no-await-in-loop await this.docker .getContainer(containerId) @@ -3617,6 +3763,7 @@ export class C2DEngineDocker extends C2DEngine { await network.remove().catch((e) => { if (!isBenignDockerError(e)) throw e }) + CORE_LOGGER.debug(`removeServiceNetwork ${serviceId}: "${ref}" removed`) } } @@ -3627,12 +3774,18 @@ export class C2DEngineDocker extends C2DEngine { private async createServiceNetwork(serviceId: string): Promise { const Name = `ocean-svc-${serviceId}` try { - return await this.docker.createNetwork({ Name }) + const network = await this.docker.createNetwork({ Name }) + CORE_LOGGER.debug(`createServiceNetwork ${serviceId}: created "${Name}"`) + return network } catch (e: any) { if (e?.statusCode !== 409) throw e CORE_LOGGER.error( `createNetwork conflict for ${Name} — removing stale network and retrying` ) + await this.logServiceDockerState( + `createServiceNetwork ${serviceId}: 409 conflict`, + serviceId + ) await this.removeServiceNetwork(serviceId) return await this.docker.createNetwork({ Name }) } @@ -3687,33 +3840,102 @@ export class C2DEngineDocker extends C2DEngine { // replaced it since our snapshot) — the current one is not known to be dead return } + // A fresh DB lease means ANOTHER process is mid-operation on this service (its + // teardown makes the container look dead); our in-memory serviceOpsInFlight can't + // see that. Skip — if the container is genuinely dead, the next tick catches it. + const lockedElsewhere = await this.db + .isServiceLocked(job.serviceId, SERVICE_LOCK_STALE_MS) + .catch(() => false) + if (lockedElsewhere) { + CORE_LOGGER.debug( + `markServiceFailed ${job.serviceId}: skipped — a lifecycle lease is held elsewhere` + ) + return + } fresh.status = ServiceStatusNumber.Error fresh.statusText = `service container exited unexpectedly: ${reason}` await this.db.updateServiceJob(fresh) CORE_LOGGER.error(`Service ${job.serviceId} container died — ${reason}`) } + // Tries to take the per-service lifecycle lock: the in-memory set serializes callers + // inside this process, the service_locks DB lease serializes across processes sharing + // the DB + Docker daemon. Returns false when someone else owns the service. + private async tryAcquireServiceLifecycleLock(serviceId: string): Promise { + // A stopped engine must not begin new lifecycle work: a replacement engine (config + // push, restart) may already be running against the same DB and daemon. + if (this.stopped) return false + if (this.serviceOpsInFlight.has(serviceId)) return false + // Reserve in-memory first — the synchronous has()→add() pair keeps concurrent local + // callers out while the async DB acquire below is in flight. + this.serviceOpsInFlight.add(serviceId) + let acquired = false + try { + acquired = await this.db.acquireServiceLock( + serviceId, + this.serviceLockHolderId, + SERVICE_LOCK_STALE_MS + ) + } catch (e: any) { + // DB trouble must not brick every lifecycle op on a single-process node — fall + // back to in-process-only locking (the pre-lease behavior). + CORE_LOGGER.warn( + `service lock DB acquire failed for ${serviceId} (falling back to in-process lock): ${e.message}` + ) + acquired = true + } + if (!acquired) { + this.serviceOpsInFlight.delete(serviceId) + CORE_LOGGER.debug( + `service lock ${serviceId}: DB lease held by another process — not acquired` + ) + } else { + CORE_LOGGER.debug( + `service lock ${serviceId}: acquired by ${this.serviceLockHolderId}` + ) + } + return acquired + } + // Takes the per-service lifecycle lock or throws: a stop must not run while the loop's // start pipeline or a restart owns the service — its by-name network removal would tear // down the docker resources the other operation is creating (and vice versa). - private acquireServiceLifecycleLock(serviceId: string): void { - if (this.serviceOpsInFlight.has(serviceId)) { + private async acquireServiceLifecycleLock(serviceId: string): Promise { + if (this.stopped) { + throw new Error( + `C2D engine is stopped — cannot run a service operation for ${serviceId}, retry shortly` + ) + } + if (!(await this.tryAcquireServiceLifecycleLock(serviceId))) { throw new Error( `Service ${serviceId} has a start/stop/restart operation in progress — retry shortly` ) } - this.serviceOpsInFlight.add(serviceId) + } + + // Releases both lock layers. Never throws: a failed DB delete only leaves a row that + // expires via SERVICE_LOCK_STALE_MS. + private async releaseServiceLifecycleLock(serviceId: string): Promise { + this.serviceOpsInFlight.delete(serviceId) + try { + await this.db.releaseServiceLock(serviceId, this.serviceLockHolderId) + CORE_LOGGER.debug( + `service lock ${serviceId}: released by ${this.serviceLockHolderId}` + ) + } catch (e: any) { + CORE_LOGGER.warn(`service lock DB release failed for ${serviceId}: ${e.message}`) + } } public override async stopService( serviceId: string, owner: string ): Promise { - this.acquireServiceLifecycleLock(serviceId) + await this.acquireServiceLifecycleLock(serviceId) // Tracked like loop-launched starts so engine stop() drains an in-flight stop too. const op = this.doStopService(serviceId, owner).finally(() => { - this.serviceOpsInFlight.delete(serviceId) this.serviceOpPromises.delete(op) + return this.releaseServiceLifecycleLock(serviceId) }) this.serviceOpPromises.add(op) return await op @@ -3726,13 +3948,25 @@ export class C2DEngineDocker extends C2DEngine { owner: string ): Promise { const [job] = await this.db.getServiceJob(serviceId, owner) - if (!job) return null + if (!job) { + CORE_LOGGER.debug(`stopService ${serviceId}: no job found for owner ${owner}`) + return null + } if ( job.status === ServiceStatusNumber.Stopped || job.status === ServiceStatusNumber.Expired - ) + ) { + CORE_LOGGER.debug( + `stopService ${serviceId}: already "${job.statusText}" — nothing to tear down` + ) return job + } + CORE_LOGGER.debug( + `stopService ${serviceId}: stopping (status=${job.statusText}, ` + + `containerId=${job.containerId || '-'}, networkId=${job.networkId || '-'})` + ) + await this.logServiceDockerState(`stop ${serviceId}: before teardown`, serviceId) job.status = ServiceStatusNumber.Stopping job.statusText = ServiceStatusText[ServiceStatusNumber.Stopping] await this.db.updateServiceJob(job) @@ -3753,20 +3987,29 @@ export class C2DEngineDocker extends C2DEngine { }) } await this.removeServiceNetwork(serviceId, job.networkId) - // Only release the reserved host ports once the container is confirmed gone — a - // port still bound by a not-removed container must not be handed to another service. - for (const ep of job.endpoints) releaseHostPort(ep.hostPort) + // Host ports deliberately NOT released: the consumer paid for the reservation + // until expiresAt and a Stopped service must be restartable anytime on the SAME + // endpoints. Only the expiry sweep releases them (after marking Expired). } catch (err: any) { cleanupError = err CORE_LOGGER.error(`stopService ${serviceId} cleanup error: ${err.message}`) } if (cleanupError) { + await this.logServiceDockerState( + `stop ${serviceId}: FAILED (${cleanupError.message}) — docker state at failure`, + serviceId + ) job.status = ServiceStatusNumber.Error job.statusText = `stop failed: ${cleanupError.message}` } else { job.status = ServiceStatusNumber.Stopped job.statusText = ServiceStatusText[ServiceStatusNumber.Stopped] + // The docker objects are confirmed gone — drop the stale ids so nothing later + // mistakes them for live resources (a restart re-creates and re-persists both). + job.containerId = '' + job.networkId = '' + CORE_LOGGER.debug(`stopService ${serviceId}: stopped, container + network removed`) } await this.db.updateServiceJob(job) return job @@ -3780,57 +4023,108 @@ export class C2DEngineDocker extends C2DEngine { newDockerEntrypoint?: string[] ): Promise { // Lifecycle lock: without it the InternalLoop's orphan-recovery (which sees the - // intermediate PullImage/BuildImage status this method persists) tears down the - // network created here mid-restart → container.start() fails with + // intermediate Restarting/PullImage/BuildImage status this method persists) tears + // down the network created here mid-restart → container.start() fails with // "network ocean-svc- not found". - this.acquireServiceLifecycleLock(serviceId) + await this.acquireServiceLifecycleLock(serviceId) + + // Validate + persist Restarting synchronously so the caller gets immediate errors + // (not found / expired / refunded), then continue in the background: the image + // pull/build can take minutes and must not block the HTTP/P2P response. Clients + // poll SERVICE_GET_STATUS and watch Restarting → … → Running (or Error). + let job: ServiceJob + try { + ;[job] = await this.db.getServiceJob(serviceId, owner) + if (!job) { + CORE_LOGGER.debug(`restartService ${serviceId}: no job found for owner ${owner}`) + return null + } + CORE_LOGGER.debug( + `restartService ${serviceId}: accepted (status=${job.statusText}, ` + + `containerId=${job.containerId || '-'}, networkId=${job.networkId || '-'}, ` + + `expiresAt=${new Date(job.expiresAt).toISOString()})` + ) + // Reject on the expiry timestamp too, not just the status: the expiry cron flips the + // status asynchronously, so a service can be past its paid window before it reads + // Expired. Restarting then would silently extend the service beyond what was paid for. + if (job.status === ServiceStatusNumber.Expired || Date.now() >= job.expiresAt) + throw new Error('Cannot restart an expired service') + // A refunded start (lock cancelled, never claimed) was never paid for — restarting + // it would run the service for free. The consumer must start a new service instead. + if (job.payment?.cancelTx && !job.payment?.claimTx) + throw new Error( + 'Cannot restart a service whose payment was refunded — start a new service' + ) + // Persist Restarting BEFORE returning: status polls flip immediately, and a crash + // from here on leaves a pending-status record the boot loop orphan-recovers + // (Restarting is in SERVICE_START_PENDING_STATUSES) instead of a bare Starting + // record that would be double-started with a second escrow lock. + job.status = ServiceStatusNumber.Restarting + job.statusText = ServiceStatusText[ServiceStatusNumber.Restarting] + await this.db.updateServiceJob(job) + } catch (e) { + await this.releaseServiceLifecycleLock(serviceId) + throw e + } + // Tracked like loop-launched starts so engine stop() drains an in-flight restart too. const op = this.doRestartService( - serviceId, - owner, + job, newUserData, newDockerCmd, newDockerEntrypoint ).finally(() => { - this.serviceOpsInFlight.delete(serviceId) this.serviceOpPromises.delete(op) + return this.releaseServiceLifecycleLock(serviceId) }) this.serviceOpPromises.add(op) - return await op + // The outcome is persisted on the job status (Running or Error + statusText); nobody + // awaits the background op, so swallow its rejection to avoid an unhandled rejection. + op.catch((e) => + CORE_LOGGER.debug(`restartService ${serviceId}: background op failed: ${e.message}`) + ) + // Snapshot for the immediate response — the DB record already says Restarting. + return job } - // The actual restart. Must only run while holding the service lifecycle lock (see - // restartService above). + // The actual restart (teardown onwards). Runs in the background after restartService + // returned; must only run while holding the service lifecycle lock, with the job + // already validated and persisted as Restarting (see restartService above). private async doRestartService( - serviceId: string, - owner: string, + job: ServiceJob, newUserData?: string, newDockerCmd?: string[], newDockerEntrypoint?: string[] ): Promise { - const [job] = await this.db.getServiceJob(serviceId, owner) - if (!job) return null - // Reject on the expiry timestamp too, not just the status: the expiry cron flips the - // status asynchronously, so a service can be past its paid window before it reads Expired. - // Restarting then would silently extend the service beyond what was paid for. - if (job.status === ServiceStatusNumber.Expired || Date.now() >= job.expiresAt) - throw new Error('Cannot restart an expired service') - - // 1. Tear down existing container + network (best-effort). Deliberately BEFORE - // persisting Starting: a bare Starting record (cleared ids) is treated as a brand-new - // start by the boot-time loop, which would create+claim a SECOND escrow lock and - // allocate NEW host ports. Crashing here instead leaves the job Running with stale - // ids — benign: the boot health check flips it to Error and a re-issued restart - // recovers on the same ports/payment (teardown 404s are swallowed). + const { serviceId } = job + + // Re-reserve the job's host ports in the in-memory allocator: a Stopped service + // released them, so without this a concurrent start could be handed the same port + // this restart is about to re-bind. For a Running service the adds are no-ops (the + // ports are still reserved). + for (const ep of job.endpoints) reserveHostPort(ep.hostPort) + + await this.logServiceDockerState(`restart ${serviceId}: before teardown`, serviceId) + + // 1. Tear down existing container + network (best-effort). The job is already + // persisted as Restarting, so a crash anywhere in this method leaves a pending-status + // record that the boot loop orphan-recovers (refund-safe: the original payment's + // claimTx is set, so recovery never touches escrow for a restart). if (job.containerId) { + CORE_LOGGER.debug(`restart ${serviceId}: removing old container ${job.containerId}`) const c = this.docker.getContainer(job.containerId) - await c.stop({ t: 10 }).catch(() => {}) - await c.remove({ force: true }).catch(() => {}) + await c.stop({ t: 10 }).catch((e) => { + CORE_LOGGER.debug(`restart ${serviceId}: old container stop: ${e.message}`) + }) + await c.remove({ force: true }).catch((e) => { + CORE_LOGGER.debug(`restart ${serviceId}: old container remove: ${e.message}`) + }) } - await this.removeServiceNetwork(serviceId, job.networkId).catch(() => {}) + await this.removeServiceNetwork(serviceId, job.networkId).catch((e) => { + CORE_LOGGER.debug(`restart ${serviceId}: old network removal: ${e.message}`) + }) + await this.logServiceDockerState(`restart ${serviceId}: after teardown`, serviceId) - job.status = ServiceStatusNumber.Starting - job.statusText = ServiceStatusText[ServiceStatusNumber.Starting] job.containerId = '' job.networkId = '' await this.db.updateServiceJob(job) @@ -3890,6 +4184,7 @@ export class C2DEngineDocker extends C2DEngine { // 6. New per-service network network = await this.createServiceNetwork(serviceId) + CORE_LOGGER.debug(`restart ${serviceId}: created network ${network.id}`) // 7. Resource constraints const { Memory, NanoCpus, DeviceRequests, CpusetCpus } = @@ -3918,7 +4213,11 @@ export class C2DEngineDocker extends C2DEngine { PidsLimit: 512 } }) + CORE_LOGGER.debug( + `restart ${serviceId}: created container ${container.id} on network ${network.id} — starting` + ) await container.start() + CORE_LOGGER.debug(`restart ${serviceId}: container ${container.id} started`) // 9. Update record — same expiresAt, same payment, new container/network. job.containerId = container.id @@ -3929,9 +4228,21 @@ export class C2DEngineDocker extends C2DEngine { job.status = ServiceStatusNumber.Running job.statusText = ServiceStatusText[ServiceStatusNumber.Running] await this.db.updateServiceJob(job) + CORE_LOGGER.info( + `restart ${serviceId}: completed — container ${container.id} Running on ` + + `ports [${job.endpoints.map((ep) => ep.hostPort).join(',')}]` + ) return job } catch (err: any) { + // Dump the daemon state BEFORE cleanup, so the log shows what the failure saw + // (e.g. whether ocean-svc- actually existed when container.start() ran). + await this.logServiceDockerState( + `restart ${serviceId}: FAILED (${err.message}) — docker state at failure`, + serviceId + ) await this.cleanupServiceDocker(container, network, serviceId) + // Ports deliberately stay reserved: the consumer paid until expiresAt and may + // restart again — the expiry sweep releases them when the window elapses. job.status = ServiceStatusNumber.Error job.statusText = String(err.message) await this.db.updateServiceJob(job) diff --git a/src/components/core/service/extendService.ts b/src/components/core/service/extendService.ts index 22f0cd364..63114e6c3 100644 --- a/src/components/core/service/extendService.ts +++ b/src/components/core/service/extendService.ts @@ -9,12 +9,10 @@ import { buildInvalidRequestMessage } from '../../httpRoutes/validateCommands.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' -import type { C2DEngine } from '../../c2d/compute_engine_base.js' import type { ComputeEnvironment } from '../../../@types/C2D/C2D.js' -import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' import { ServiceStatusNumber } from '../../../@types/C2D/ServiceOnDemand.js' import { validateAccess } from '../compute/startCompute.js' -import { toPublicServiceJob } from './utils.js' +import { findServiceJobAndEngine, toPublicServiceJob } from './utils.js' export class ServiceExtendHandler extends CommandHandler { validate(command: ServiceExtendCommand): ValidateParams { @@ -51,21 +49,24 @@ export class ServiceExtendHandler extends CommandHandler { status: { httpStatus: 503, error: 'Compute engines not configured' } } - // Find job - let job: ServiceJob | null = null - let engine: C2DEngine | null = null - for (const eng of engines.getAllEngines()) { - const [found] = await eng.db.getServiceJob(task.serviceId, task.consumerAddress) - if (found) { - job = found - engine = eng - break - } - } - if (!job || !engine) + // Find the job and the engine that owns it (by clusterHash — see helper) + const { job, engine } = await findServiceJobAndEngine( + engines, + task.serviceId, + task.consumerAddress + ) + if (!job) return buildInvalidParametersResponse( buildInvalidRequestMessage('Service job not found: ' + task.serviceId) ) + if (!engine) + return { + stream: null, + status: { + httpStatus: 500, + error: `No compute engine owns service ${task.serviceId} (cluster ${job.clusterHash}) — the node's compute configuration may have changed` + } + } // Ownership check if (job.owner.toLowerCase() !== task.consumerAddress.toLowerCase()) diff --git a/src/components/core/service/getStreamableLogs.ts b/src/components/core/service/getStreamableLogs.ts index 8973ff696..fb6ebcfdf 100644 --- a/src/components/core/service/getStreamableLogs.ts +++ b/src/components/core/service/getStreamableLogs.ts @@ -9,9 +9,7 @@ import { buildInvalidRequestMessage } from '../../httpRoutes/validateCommands.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' -import type { C2DEngine } from '../../c2d/compute_engine_base.js' -import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' -import { parseSinceParam } from './utils.js' +import { findServiceJobAndEngine, parseSinceParam } from './utils.js' export class ServiceGetStreamableLogsHandler extends CommandHandler { validate(command: ServiceGetStreamableLogsCommand): ValidateParams { @@ -50,21 +48,24 @@ export class ServiceGetStreamableLogsHandler extends CommandHandler { status: { httpStatus: 503, error: 'Compute engines not configured' } } - // Find job across all engines by serviceId + owner - let job: ServiceJob | null = null - let engine: C2DEngine | null = null - for (const eng of engines.getAllEngines()) { - const [found] = await eng.db.getServiceJob(task.serviceId, task.consumerAddress) - if (found) { - job = found - engine = eng - break - } - } - if (!job || !engine) + // Find the job and the engine that owns it (by clusterHash — see helper) + const { job, engine } = await findServiceJobAndEngine( + engines, + task.serviceId, + task.consumerAddress + ) + if (!job) return buildInvalidParametersResponse( buildInvalidRequestMessage('Service job not found: ' + task.serviceId) ) + if (!engine) + return { + stream: null, + status: { + httpStatus: 500, + error: `No compute engine owns service ${task.serviceId} (cluster ${job.clusterHash}) — the node's compute configuration may have changed` + } + } if (job.owner.toLowerCase() !== task.consumerAddress.toLowerCase()) return { stream: null, status: { httpStatus: 401, error: 'Not the service owner' } } diff --git a/src/components/core/service/restartService.ts b/src/components/core/service/restartService.ts index bc3b03464..ed5bcd9c3 100644 --- a/src/components/core/service/restartService.ts +++ b/src/components/core/service/restartService.ts @@ -9,12 +9,10 @@ import { buildInvalidRequestMessage } from '../../httpRoutes/validateCommands.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' -import type { C2DEngine } from '../../c2d/compute_engine_base.js' import type { ComputeEnvironment } from '../../../@types/C2D/C2D.js' -import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' import { ServiceStatusNumber } from '../../../@types/C2D/ServiceOnDemand.js' import { validateAccess } from '../compute/startCompute.js' -import { decryptUserData, toPublicServiceJob } from './utils.js' +import { decryptUserData, findServiceJobAndEngine, toPublicServiceJob } from './utils.js' export class ServiceRestartHandler extends CommandHandler { validate(command: ServiceRestartCommand): ValidateParams { @@ -42,21 +40,24 @@ export class ServiceRestartHandler extends CommandHandler { status: { httpStatus: 503, error: 'Compute engines not configured' } } - // Find job across all engines - let job: ServiceJob | null = null - let engine: C2DEngine | null = null - for (const eng of engines.getAllEngines()) { - const [found] = await eng.db.getServiceJob(task.serviceId, task.consumerAddress) - if (found) { - job = found - engine = eng - break - } - } - if (!job || !engine) + // Find the job and the engine that owns it (by clusterHash — see helper) + const { job, engine } = await findServiceJobAndEngine( + engines, + task.serviceId, + task.consumerAddress + ) + if (!job) return buildInvalidParametersResponse( buildInvalidRequestMessage('Service job not found: ' + task.serviceId) ) + if (!engine) + return { + stream: null, + status: { + httpStatus: 500, + error: `No compute engine owns service ${task.serviceId} (cluster ${job.clusterHash}) — the node's compute configuration may have changed` + } + } // Ownership check if (job.owner.toLowerCase() !== task.consumerAddress.toLowerCase()) @@ -107,6 +108,11 @@ export class ServiceRestartHandler extends CommandHandler { } try { + // Asynchronous, like SERVICE_START: engine.restartService validates, persists the + // job as Restarting and returns immediately — the teardown + image pull/build + + // new container run in the background (an image pull can take minutes and must + // not block the HTTP/P2P response). Clients poll SERVICE_GET_STATUS and watch + // Restarting → … → Running (or an Error status with the failure reason). const restarted = await engine.restartService( task.serviceId, task.consumerAddress, diff --git a/src/components/core/service/stopService.ts b/src/components/core/service/stopService.ts index 90587a5f7..52330ae1a 100644 --- a/src/components/core/service/stopService.ts +++ b/src/components/core/service/stopService.ts @@ -9,9 +9,7 @@ import { buildInvalidRequestMessage } from '../../httpRoutes/validateCommands.js' import { CORE_LOGGER } from '../../../utils/logging/common.js' -import type { C2DEngine } from '../../c2d/compute_engine_base.js' -import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' -import { toPublicServiceJob } from './utils.js' +import { findServiceJobAndEngine, toPublicServiceJob } from './utils.js' export class ServiceStopHandler extends CommandHandler { validate(command: ServiceStopCommand): ValidateParams { @@ -38,21 +36,24 @@ export class ServiceStopHandler extends CommandHandler { status: { httpStatus: 503, error: 'Compute engines not configured' } } - // Find job across all engines by serviceId + owner - let job: ServiceJob | null = null - let engine: C2DEngine | null = null - for (const eng of engines.getAllEngines()) { - const [found] = await eng.db.getServiceJob(task.serviceId, task.consumerAddress) - if (found) { - job = found - engine = eng - break - } - } - if (!job || !engine) + // Find the job and the engine that owns it (by clusterHash — see helper) + const { job, engine } = await findServiceJobAndEngine( + engines, + task.serviceId, + task.consumerAddress + ) + if (!job) return buildInvalidParametersResponse( buildInvalidRequestMessage('Service job not found: ' + task.serviceId) ) + if (!engine) + return { + stream: null, + status: { + httpStatus: 500, + error: `No compute engine owns service ${task.serviceId} (cluster ${job.clusterHash}) — the node's compute configuration may have changed` + } + } if (job.owner.toLowerCase() !== task.consumerAddress.toLowerCase()) return { stream: null, status: { httpStatus: 401, error: 'Not the service owner' } } diff --git a/src/components/core/service/utils.ts b/src/components/core/service/utils.ts index 95b051924..278db5101 100644 --- a/src/components/core/service/utils.ts +++ b/src/components/core/service/utils.ts @@ -3,6 +3,27 @@ import type { ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' import { EncryptMethod } from '../../../@types/fileObject.js' import type { KeyManager } from '../../KeyManager/index.js' import type { C2DDatabase } from '../../database/C2DDatabase.js' +import type { C2DEngine } from '../../c2d/compute_engine_base.js' +import type { C2DEngines } from '../../c2d/compute_engines.js' + +// Looks up a service job and resolves the engine that OWNS it (by clusterHash). Every +// engine shares the same C2DDatabase, so any engine's db returns the job — taking the +// first engine that "finds" it (the old pattern) breaks on nodes with several docker +// engines: the wrong engine's lifecycle lock and InternalLoop would not protect the +// job, resurrecting the teardown-mid-restart race. engine === null with a non-null job +// means no configured engine matches the job's clusterHash (node config changed). +export async function findServiceJobAndEngine( + engines: C2DEngines, + serviceId: string, + owner?: string +): Promise<{ job: ServiceJob | null; engine: C2DEngine | null }> { + const all = engines.getAllEngines() + if (all.length === 0) return { job: null, engine: null } + const [job] = await all[0].db.getServiceJob(serviceId, owner) + if (!job) return { job: null, engine: null } + const engine = all.find((e) => e.getC2DConfig().hash === job.clusterHash) ?? null + return { job, engine } +} // Converts the decrypted userData object into a flat container env-var map (stringified values). export function userDataToEnv(userData: Record): Record { @@ -104,6 +125,13 @@ export function releaseHostPort(port: number): void { allocatedPorts.delete(port) } +// Marks an already-assigned port as reserved (idempotent). Used by restart, which +// re-binds the ports recorded on the job: after a stop (or an Error path) released +// them, they must go back into the set before the container binds them again. +export function reserveHostPort(port: number): void { + allocatedPorts.add(port) +} + function isPortFree(port: number): Promise { return new Promise((resolve) => { const s = net.createServer() diff --git a/src/components/database/C2DDatabase.ts b/src/components/database/C2DDatabase.ts index 68b3acea1..b19d30dea 100755 --- a/src/components/database/C2DDatabase.ts +++ b/src/components/database/C2DDatabase.ts @@ -32,6 +32,7 @@ export class C2DDatabase extends AbstractDatabase { await this.provider.createTable() await this.provider.createImageTable() await this.provider.createServiceTable() + await this.provider.createServiceLocksTable() return this })() as unknown as C2DDatabase @@ -99,6 +100,28 @@ export class C2DDatabase extends AbstractDatabase { return await this.provider.getPendingServiceStarts(clusterHash) } + // ── Service lifecycle locks (cross-process; see sqliteCompute.ts) ─────── + + async acquireServiceLock( + serviceId: string, + holder: string, + staleMs: number + ): Promise { + return await this.provider.acquireServiceLock(serviceId, holder, staleMs) + } + + async releaseServiceLock(serviceId: string, holder: string): Promise { + return await this.provider.releaseServiceLock(serviceId, holder) + } + + async refreshServiceLocks(holder: string): Promise { + return await this.provider.refreshServiceLocks(holder) + } + + async isServiceLocked(serviceId: string, staleMs: number): Promise { + return await this.provider.isServiceLocked(serviceId, staleMs) + } + async deleteJob(jobId: string): Promise { return await this.provider.deleteJob(jobId) } diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index 5623edd2c..a0c8368ef 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -4,7 +4,11 @@ import { C2DStatusText, type DBComputeJob } from '../../@types/C2D/C2D.js' -import { ServiceStatusNumber, type ServiceJob } from '../../@types/C2D/ServiceOnDemand.js' +import { + ServiceStatusNumber, + SERVICE_START_PENDING_STATUSES, + type ServiceJob +} from '../../@types/C2D/ServiceOnDemand.js' import sqlite3, { RunResult } from 'sqlite3' import { DATABASE_LOGGER } from '../../utils/logging/common.js' import { create256Hash } from '../../utils/crypt.js' @@ -153,6 +157,113 @@ export class SQLiteCompute implements ComputeDatabaseProvider { // ── Service-on-Demand jobs ────────────────────────────────────────── + // Cross-process lifecycle locks for service jobs. A row = an exclusive start/stop/ + // restart operation in flight on serviceId, held by `holder` (a per-process id). Rows + // are heartbeated (acquiredAt refreshed) while the operation runs; a row whose + // acquiredAt is older than the staleness window is a crashed holder and may be stolen. + // This extends the engine's in-memory serviceOpsInFlight guarantee to setups where + // several node processes share the same DB file + Docker daemon (e.g. a stale + // container still running during a redeploy) — in-memory sets cannot see each other. + createServiceLocksTable(): Promise { + const createTableSQL = ` + CREATE TABLE IF NOT EXISTS service_locks ( + serviceId TEXT PRIMARY KEY, + holder TEXT NOT NULL, + acquiredAt INTEGER NOT NULL + ); + ` + return new Promise((resolve, reject) => { + this.db.run(createTableSQL, (err) => { + if (err) { + DATABASE_LOGGER.error('Could not create service_locks table: ' + err.message) + reject(err) + } else { + resolve() + } + }) + }) + } + + // Atomically takes the lock for serviceId: inserts a fresh row, or steals one whose + // acquiredAt is older than staleMs (crashed holder). The single upsert statement is + // the atomicity guarantee — two processes racing it can never both see success. + acquireServiceLock( + serviceId: string, + holder: string, + staleMs: number + ): Promise { + const now = Date.now() + const upsertSQL = ` + INSERT INTO service_locks (serviceId, holder, acquiredAt) VALUES (?, ?, ?) + ON CONFLICT(serviceId) DO UPDATE + SET holder = excluded.holder, acquiredAt = excluded.acquiredAt + WHERE service_locks.acquiredAt <= ?; + ` + return new Promise((resolve, reject) => { + this.db.run( + upsertSQL, + [serviceId, holder, now, now - staleMs], + function (this: RunResult, err: Error | null) { + if (err) { + DATABASE_LOGGER.error(`Could not acquire service lock: ${err.message}`) + reject(err) + } else { + resolve(this.changes === 1) + } + } + ) + }) + } + + // Releases only a lock we still hold — a stale lock stolen by another process must + // not be deleted out from under its new holder. + releaseServiceLock(serviceId: string, holder: string): Promise { + const deleteSQL = `DELETE FROM service_locks WHERE serviceId = ? AND holder = ?;` + return new Promise((resolve, reject) => { + this.db.run(deleteSQL, [serviceId, holder], (err) => { + if (err) { + DATABASE_LOGGER.error(`Could not release service lock: ${err.message}`) + reject(err) + } else { + resolve() + } + }) + }) + } + + // Heartbeat: re-stamps every lock this holder owns so long operations (multi-minute + // image pulls/builds) are not stolen as stale. + refreshServiceLocks(holder: string): Promise { + const updateSQL = `UPDATE service_locks SET acquiredAt = ? WHERE holder = ?;` + return new Promise((resolve, reject) => { + this.db.run(updateSQL, [Date.now(), holder], (err) => { + if (err) { + DATABASE_LOGGER.error(`Could not refresh service locks: ${err.message}`) + reject(err) + } else { + resolve() + } + }) + }) + } + + // True while ANY process holds a fresh lock on serviceId — used by read-only + // observers (e.g. the container health check) to avoid judging a service that + // another process is mid-way through restarting. + isServiceLocked(serviceId: string, staleMs: number): Promise { + const selectSQL = `SELECT acquiredAt FROM service_locks WHERE serviceId = ?;` + return new Promise((resolve, reject) => { + this.db.get(selectSQL, [serviceId], (err, row: any) => { + if (err) { + DATABASE_LOGGER.error(`Could not read service lock: ${err.message}`) + reject(err) + } else { + resolve(!!row && row.acquiredAt > Date.now() - staleMs) + } + }) + }) + } + createServiceTable(): Promise { const createTableSQL = ` CREATE TABLE IF NOT EXISTS service_jobs ( @@ -265,11 +376,13 @@ export class SQLiteCompute implements ComputeDatabaseProvider { } getRunningServiceJobs(clusterHash?: string): Promise { - // All pre-terminal statuses are "active": a service reserves its resources from the moment - // its record is created (Starting) through the whole start pipeline (Locking, image, Claiming) - // until it is Running, and while Stopping. Error is included too: a service whose container - // died on its own keeps its resources/ports reserved (consumer already paid for them) until - // it is explicitly restarted/stopped, or swept by getExpiredServiceJobs once past expiresAt. + // Every status before Expired is "active": the consumer paid for the resources for a + // TIME WINDOW, so the reservation holds from the moment the record is created + // (Starting) through the whole start pipeline (Locking, image, Claiming), while + // Running/Restarting/Stopping, through Error (container died on its own) AND through + // an explicit Stopped — a stopped service can be restarted anytime on the same + // resources until expiresAt. Only the expiry sweep (→ Expired) releases the + // reservation; nothing else may free it inside the paid window. const activeStatuses = [ ServiceStatusNumber.Starting, ServiceStatusNumber.Locking, @@ -277,7 +390,9 @@ export class SQLiteCompute implements ComputeDatabaseProvider { ServiceStatusNumber.BuildImage, ServiceStatusNumber.Claiming, ServiceStatusNumber.Running, + ServiceStatusNumber.Restarting, ServiceStatusNumber.Stopping, + ServiceStatusNumber.Stopped, ServiceStatusNumber.Error ] const placeholders = activeStatuses.map(() => '?').join(',') @@ -300,11 +415,15 @@ export class SQLiteCompute implements ComputeDatabaseProvider { } getExpiredServiceJobs(clusterHash?: string): Promise { - // Running AND Error: an Error'd service (e.g. container died on its own) still holds its - // resources/ports reserved for a possible restart, but once past expiresAt it must be - // swept and released just like a normally-running one — otherwise an abandoned Error - // service would leak its reservation forever. - const expirableStatuses = [ServiceStatusNumber.Running, ServiceStatusNumber.Error] + // Running, Error AND Stopped all still hold their paid reservation (see activeStatuses + // above), so all three must be swept once past expiresAt: the sweep is the ONLY place + // the reservation is released. Without it an abandoned Error or Stopped service would + // keep its resources/ports forever and still read as restartable. + const expirableStatuses = [ + ServiceStatusNumber.Running, + ServiceStatusNumber.Error, + ServiceStatusNumber.Stopped + ] const placeholders = expirableStatuses.map(() => '?').join(',') const params: Array = [...expirableStatuses, Date.now()] let selectSQL = `SELECT * FROM service_jobs WHERE status IN (${placeholders}) AND expiresAt <= ?` @@ -328,13 +447,7 @@ export class SQLiteCompute implements ComputeDatabaseProvider { // Starting = fresh (handler just created it); the intermediate states are picked up too so // the loop can resume / orphan-recover them after a node restart. getPendingServiceStarts(clusterHash?: string): Promise { - const startStatuses = [ - ServiceStatusNumber.Starting, - ServiceStatusNumber.Locking, - ServiceStatusNumber.PullImage, - ServiceStatusNumber.BuildImage, - ServiceStatusNumber.Claiming - ] + const startStatuses = SERVICE_START_PENDING_STATUSES const placeholders = startStatuses.map(() => '?').join(',') const params: Array = [...startStatuses] let selectSQL = `SELECT * FROM service_jobs WHERE status IN (${placeholders})` diff --git a/src/test/integration/services.test.ts b/src/test/integration/services.test.ts index bfc5ea305..449f2d39b 100644 --- a/src/test/integration/services.test.ts +++ b/src/test/integration/services.test.ts @@ -319,11 +319,12 @@ describe('********** Service on Demand', () => { ) dbconn = await Database.init(config.dbConfig) - // Clean stale running service jobs so prior runs don't consume shared resources. + // Clean stale service jobs so prior runs don't consume shared resources. Expired is + // the only status that releases the reservation (Stopped still counts as active). const staleServices = await dbconn.c2d.getRunningServiceJobs() for (const svc of staleServices) { - svc.status = ServiceStatusNumber.Stopped - svc.statusText = 'Stopped' + svc.status = ServiceStatusNumber.Expired + svc.statusText = 'Expired' await dbconn.c2d.updateServiceJob(svc) } const staleJobs = await dbconn.c2d.getRunningJobs() diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 8ff5a4651..68e5c7424 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -41,6 +41,7 @@ import { C2DEngineDocker } from '../../components/c2d/compute_engine_docker.js' import { ServiceStatusNumber } from '../../@types/C2D/ServiceOnDemand.js' +import { allocateHostPort, releaseHostPort } from '../../components/core/service/utils.js' import { C2DDockerConfigSchema } from '../../utils/config/schemas.js' import { ValidateParams } from '../../components/httpRoutes/validateCommands.js' import { Readable } from 'stream' @@ -1604,6 +1605,8 @@ describe('service start/restart Docker cleanup on failure', function () { getNetwork: sinon.stub().returns(network) } const job = makeStartingJob({ serviceId: 'svc-1' }) + // the pipeline re-reads the job under the lifecycle lock before acting + engine.db.getServiceJob = sinon.stub().resolves([job]) await engine.processServiceStart(job) // never throws — failures are persisted as status @@ -1622,6 +1625,7 @@ describe('service start/restart Docker cleanup on failure', function () { getNetwork: sinon.stub().returns(network) } const job = makeStartingJob({ serviceId: 'svc-2' }) + engine.db.getServiceJob = sinon.stub().resolves([job]) await engine.processServiceStart(job) @@ -1630,6 +1634,13 @@ describe('service start/restart Docker cleanup on failure', function () { ) expect(network.remove.called, 'network.remove should be called').to.equal(true) expect(job.status).to.equal(ServiceStatusNumber.Error) + // The payment was claimed before the container step, so the Error job keeps its + // host ports reserved (restartable on the same endpoints until expiresAt): the + // allocator must refuse to hand the port out again. + expect(job.endpoints.length).to.be.greaterThan(0) + const reservedPort = job.endpoints[0].hostPort + await expectRejects(allocateHostPort(reservedPort, reservedPort), 'No free host port') + releaseHostPort(reservedPort) // don't leak the reservation into other tests }) it('processServiceStart refunds (cancelLock) and marks PullImageFailed when the image pull fails', async function () { @@ -1640,6 +1651,7 @@ describe('service start/restart Docker cleanup on failure', function () { getNetwork: sinon.stub().returns(network) } const job = makeStartingJob({ serviceId: 'svc-img' }) + engine.db.getServiceJob = sinon.stub().resolves([job]) await engine.processServiceStart(job) @@ -1663,6 +1675,7 @@ describe('service start/restart Docker cleanup on failure', function () { getNetwork: sinon.stub().returns(network) } const job = makeStartingJob({ serviceId: 'svc-lock' }) + engine.db.getServiceJob = sinon.stub().resolves([job]) await engine.processServiceStart(job) @@ -1691,6 +1704,7 @@ describe('service start/restart Docker cleanup on failure', function () { cancelTx: '' } }) + engine.db.getServiceJob = sinon.stub().resolves([job]) await engine.processServiceStart(job) @@ -1730,12 +1744,15 @@ describe('service start/restart Docker cleanup on failure', function () { getNetwork: sinon.stub().returns(network) } - await expectRejects( - engine.restartService('svc-3', '0xowner', undefined), - 'createContainer failed' - ) + // restart is async: the call returns the Restarting snapshot; the failure surfaces + // on the persisted job status once the background op settles. + const snapshot = await engine.restartService('svc-3', '0xowner', undefined) + expect(snapshot.status).to.equal(ServiceStatusNumber.Restarting) + await Promise.allSettled([...engine.serviceOpPromises]) expect(network.remove.called, 'network.remove should be called').to.equal(true) + expect(existingJob.status).to.equal(ServiceStatusNumber.Error) + expect(existingJob.statusText).to.contain('createContainer failed') }) function makeRunningJobWithCmd(overrides: any = {}) { @@ -1774,19 +1791,20 @@ describe('service start/restart Docker cleanup on failure', function () { getNetwork: sinon.stub().returns(network) } - const result = await engine.restartService( + await engine.restartService( 'svc-cmd', '0xowner', undefined, ['new', 'cmd'], ['/new-entrypoint'] ) + await Promise.allSettled([...engine.serviceOpPromises]) const createArgs = engine.docker.createContainer.firstCall.args[0] expect(createArgs.Cmd).to.deep.equal(['new', 'cmd']) expect(createArgs.Entrypoint).to.deep.equal(['/new-entrypoint']) - expect(result.dockerCmd).to.deep.equal(['new', 'cmd']) - expect(result.dockerEntrypoint).to.deep.equal(['/new-entrypoint']) + expect(existingJob.dockerCmd).to.deep.equal(['new', 'cmd']) + expect(existingJob.dockerEntrypoint).to.deep.equal(['/new-entrypoint']) }) it('restartService reuses the stored dockerCmd/dockerEntrypoint when none are supplied', async function () { @@ -1799,13 +1817,14 @@ describe('service start/restart Docker cleanup on failure', function () { getNetwork: sinon.stub().returns(network) } - const result = await engine.restartService('svc-cmd', '0xowner', undefined) + await engine.restartService('svc-cmd', '0xowner', undefined) + await Promise.allSettled([...engine.serviceOpPromises]) const createArgs = engine.docker.createContainer.firstCall.args[0] expect(createArgs.Cmd).to.deep.equal(['old', 'cmd']) expect(createArgs.Entrypoint).to.deep.equal(['/old-entrypoint']) - expect(result.dockerCmd).to.deep.equal(['old', 'cmd']) - expect(result.dockerEntrypoint).to.deep.equal(['/old-entrypoint']) + expect(existingJob.dockerCmd).to.deep.equal(['old', 'cmd']) + expect(existingJob.dockerEntrypoint).to.deep.equal(['/old-entrypoint']) }) it('restartService clears dockerCmd/dockerEntrypoint when explicitly given an empty array', async function () { @@ -1818,12 +1837,13 @@ describe('service start/restart Docker cleanup on failure', function () { getNetwork: sinon.stub().returns(network) } - const result = await engine.restartService('svc-cmd', '0xowner', undefined, [], []) + await engine.restartService('svc-cmd', '0xowner', undefined, [], []) + await Promise.allSettled([...engine.serviceOpPromises]) const createArgs = engine.docker.createContainer.firstCall.args[0] expect(createArgs.Cmd).to.equal(undefined) expect(createArgs.Entrypoint).to.equal(undefined) - expect(result.dockerCmd).to.deep.equal([]) - expect(result.dockerEntrypoint).to.deep.equal([]) + expect(existingJob.dockerCmd).to.deep.equal([]) + expect(existingJob.dockerEntrypoint).to.deep.equal([]) }) }) diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index b09d58413..6a20d7294 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -94,9 +94,11 @@ function buildFakes(opts: FakeOpts = {}) { }, escrow, getComputeEnvironments: sinon.stub().resolves([env]), - getC2DConfig: sinon - .stub() - .returns({ connection: { serviceOnDemand: { maxDurationSeconds: 86400 } } }), + // hash must match makeJob's clusterHash: handlers resolve the owning engine by it + getC2DConfig: sinon.stub().returns({ + hash: 'hash-1', + connection: { serviceOnDemand: { maxDurationSeconds: 86400 } } + }), calculateResourcesCost: sinon .stub() .returns(opts.cost === undefined ? 10 : opts.cost), diff --git a/src/test/unit/service/serviceJobsDatabase.test.ts b/src/test/unit/service/serviceJobsDatabase.test.ts index fc21c39d6..f1945be00 100644 --- a/src/test/unit/service/serviceJobsDatabase.test.ts +++ b/src/test/unit/service/serviceJobsDatabase.test.ts @@ -141,6 +141,51 @@ describe('Service Jobs Database', () => { await tearDownEnvironment(envOverrides) }) + describe('service lifecycle locks (cross-process lease)', () => { + const STALE_MS = 60_000 + + it('grants the lock to exactly one holder and rejects a second acquire', async () => { + const serviceId = `lock-svc-${Date.now()}-a` + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + // second process: fresh lock held by A → refused + expect(await db.acquireServiceLock(serviceId, 'proc-B', STALE_MS)).to.equal(false) + // A releasing frees it for B + await db.releaseServiceLock(serviceId, 'proc-A') + expect(await db.acquireServiceLock(serviceId, 'proc-B', STALE_MS)).to.equal(true) + await db.releaseServiceLock(serviceId, 'proc-B') + }) + + it('steals a stale lock (crashed holder) but not a fresh one', async () => { + const serviceId = `lock-svc-${Date.now()}-b` + // staleMs = 0 makes A's row instantly stale — simulates a dead process + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + expect(await db.acquireServiceLock(serviceId, 'proc-B', 0)).to.equal(true) + // now B's row is fresh: A must not get it back with a normal window + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(false) + await db.releaseServiceLock(serviceId, 'proc-B') + }) + + it('release is holder-scoped: a stale-steal victim cannot delete the new lock', async () => { + const serviceId = `lock-svc-${Date.now()}-c` + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + expect(await db.acquireServiceLock(serviceId, 'proc-B', 0)).to.equal(true) // steal + // A (which lost the lock) releasing must be a no-op for B's row + await db.releaseServiceLock(serviceId, 'proc-A') + expect(await db.isServiceLocked(serviceId, STALE_MS)).to.equal(true) + await db.releaseServiceLock(serviceId, 'proc-B') + expect(await db.isServiceLocked(serviceId, STALE_MS)).to.equal(false) + }) + + it('refreshServiceLocks re-stamps a holder’s rows so they stay unstealable', async () => { + const serviceId = `lock-svc-${Date.now()}-d` + expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + await db.refreshServiceLocks('proc-A') + // B considers anything older than 5s stale — A's row was just re-stamped + expect(await db.acquireServiceLock(serviceId, 'proc-B', 5_000)).to.equal(false) + await db.releaseServiceLock(serviceId, 'proc-A') + }) + }) + it('inserts and reads back a service job by serviceId', async () => { const job = makeServiceJob() await db.newServiceJob(job) @@ -180,6 +225,7 @@ describe('Service Jobs Database', () => { const starting = makeServiceJob({ status: ServiceStatusNumber.Starting }) const locking = makeServiceJob({ status: ServiceStatusNumber.Locking }) const claiming = makeServiceJob({ status: ServiceStatusNumber.Claiming }) + const restarting = makeServiceJob({ status: ServiceStatusNumber.Restarting }) const error = makeServiceJob({ status: ServiceStatusNumber.Error }) const stopped = makeServiceJob({ status: ServiceStatusNumber.Stopped }) const otherCluster = makeServiceJob({ @@ -190,6 +236,7 @@ describe('Service Jobs Database', () => { await db.newServiceJob(starting) await db.newServiceJob(locking) await db.newServiceJob(claiming) + await db.newServiceJob(restarting) await db.newServiceJob(error) await db.newServiceJob(stopped) await db.newServiceJob(otherCluster) @@ -201,10 +248,14 @@ describe('Service Jobs Database', () => { // the new start-pipeline states must reserve resources too expect(ids).to.include(locking.serviceId) expect(ids).to.include(claiming.serviceId) + // a mid-restart service keeps holding its resources + expect(ids).to.include(restarting.serviceId) // a service whose container died on its own still reserves its resources until - // restarted/stopped/expired + // restarted or expired expect(ids).to.include(error.serviceId) - expect(ids).to.not.include(stopped.serviceId) + // an explicitly-stopped service KEEPS its reservation too: the consumer paid for + // the whole window and may restart it anytime — only Expired releases resources + expect(ids).to.include(stopped.serviceId) expect(ids).to.not.include(otherCluster.serviceId) }) @@ -212,6 +263,7 @@ describe('Service Jobs Database', () => { const starting = makeServiceJob({ status: ServiceStatusNumber.Starting }) const locking = makeServiceJob({ status: ServiceStatusNumber.Locking }) const claiming = makeServiceJob({ status: ServiceStatusNumber.Claiming }) + const restarting = makeServiceJob({ status: ServiceStatusNumber.Restarting }) const running = makeServiceJob({ status: ServiceStatusNumber.Running }) const stopped = makeServiceJob({ status: ServiceStatusNumber.Stopped }) const otherCluster = makeServiceJob({ @@ -221,6 +273,7 @@ describe('Service Jobs Database', () => { await db.newServiceJob(starting) await db.newServiceJob(locking) await db.newServiceJob(claiming) + await db.newServiceJob(restarting) await db.newServiceJob(running) await db.newServiceJob(stopped) await db.newServiceJob(otherCluster) @@ -230,12 +283,14 @@ describe('Service Jobs Database', () => { expect(ids).to.include(starting.serviceId) expect(ids).to.include(locking.serviceId) expect(ids).to.include(claiming.serviceId) + // a crash mid-restart must be orphan-recovered at boot exactly like a crash mid-start + expect(ids).to.include(restarting.serviceId) expect(ids).to.not.include(running.serviceId) // already started expect(ids).to.not.include(stopped.serviceId) expect(ids).to.not.include(otherCluster.serviceId) }) - it('getExpiredServiceJobs returns Running and Error jobs past expiry', async () => { + it('getExpiredServiceJobs returns Running, Error and Stopped jobs past expiry', async () => { const expired = makeServiceJob({ status: ServiceStatusNumber.Running, expiresAt: Date.now() - 1000 @@ -248,14 +303,19 @@ describe('Service Jobs Database', () => { status: ServiceStatusNumber.Running, expiresAt: Date.now() + 3600_000 }) - const expiredButStopped = makeServiceJob({ + const expiredStopped = makeServiceJob({ status: ServiceStatusNumber.Stopped, expiresAt: Date.now() - 1000 }) + const futureStopped = makeServiceJob({ + status: ServiceStatusNumber.Stopped, + expiresAt: Date.now() + 3600_000 + }) await db.newServiceJob(expired) await db.newServiceJob(expiredError) await db.newServiceJob(future) - await db.newServiceJob(expiredButStopped) + await db.newServiceJob(expiredStopped) + await db.newServiceJob(futureStopped) const expiredJobs = await db.getExpiredServiceJobs(CLUSTER_HASH) const ids = expiredJobs.map((j) => j.serviceId) @@ -263,8 +323,11 @@ describe('Service Jobs Database', () => { // an abandoned Error'd service must still be swept once past expiresAt, or its // reserved resources/ports would leak forever expect(ids).to.include(expiredError.serviceId) + // a Stopped service past its paid window must flip to Expired instead of sitting + // at Stopped forever (where it would read as restartable) + expect(ids).to.include(expiredStopped.serviceId) expect(ids).to.not.include(future.serviceId) - expect(ids).to.not.include(expiredButStopped.serviceId) + expect(ids).to.not.include(futureStopped.serviceId) }) describe('shared resource accounting (compute + service)', () => { @@ -278,10 +341,11 @@ describe('Service Jobs Database', () => { ['gpu0', 2] ]) ) - // Clean slate: stop any leftover running jobs from earlier tests by marking them Stopped. + // Clean slate: retire any leftover jobs from earlier tests by marking them Expired — + // the only status that releases the reservation (Stopped still counts as active). const leftovers = await db.getRunningServiceJobs(CLUSTER_HASH) for (const j of leftovers) { - j.status = ServiceStatusNumber.Stopped + j.status = ServiceStatusNumber.Expired await db.updateServiceJob(j) } }) diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts index 0ff6a43da..69d0b1fcd 100644 --- a/src/test/unit/service/serviceRestartRace.test.ts +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -2,6 +2,11 @@ import { expect, assert } from 'chai' import sinon from 'sinon' import { C2DEngineDocker } from '../../../components/c2d/compute_engine_docker.js' import { ServiceStatusNumber, ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' +import { + allocateHostPort, + releaseHostPort, + reserveHostPort +} from '../../../components/core/service/utils.js' const OWNER = '0x0000000000000000000000000000000000000001' const SERVICE_ID = 'svc-race-1' @@ -72,8 +77,14 @@ function makeEngine(): any { updateServiceJob: sinon.stub().resolves(1), getRunningJobs: sinon.stub().resolves([]), getPendingServiceStarts: sinon.stub().resolves([]), - getExpiredServiceJobs: sinon.stub().resolves([]) + getExpiredServiceJobs: sinon.stub().resolves([]), + // cross-process lifecycle lease (service_locks table) — free by default + acquireServiceLock: sinon.stub().resolves(true), + releaseServiceLock: sinon.stub().resolves(undefined), + refreshServiceLocks: sinon.stub().resolves(undefined), + isServiceLocked: sinon.stub().resolves(false) } + engine.serviceLockHolderId = 'test-holder' engine.cpuAllocations = new Map() engine.serviceOpsInFlight = new Set() engine.serviceOpPromises = new Set() @@ -101,7 +112,7 @@ function stubContainer(overrides: Record = {}) { describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { afterEach(() => sinon.restore()) - it('restartService holds the lock while the image pull is pending and releases it after', async () => { + it('restartService returns Restarting immediately and holds the lock until the background op settles', async () => { const engine = makeEngine() const job = makeJob() engine.db.getServiceJob.resolves([job]) @@ -116,30 +127,59 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { resolvePull = resolve }) - const restart = engine.restartService(SERVICE_ID, OWNER) - await flush() + // Non-blocking: resolves as soon as the job is validated + persisted Restarting, + // while the teardown/pull/start continue in the background under the lock. + const snapshot = await engine.restartService(SERVICE_ID, OWNER) + expect(snapshot.status).to.equal(ServiceStatusNumber.Restarting) expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(true) + await flush() // let the background op reach the pending pull resolvePull!() - const restarted = await restart + await Promise.allSettled([...engine.serviceOpPromises]) expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) - expect(restarted.status).to.equal(ServiceStatusNumber.Running) - expect(restarted.containerId).to.equal('newc') + expect(job.status).to.equal(ServiceStatusNumber.Running) + expect(job.containerId).to.equal('newc') }) - it('restartService releases the lock on the failure path too', async () => { + it('restartService releases the lock and persists Error on the failure path', async () => { const engine = makeEngine() - engine.db.getServiceJob.resolves([makeJob()]) + const job = makeJob() + engine.db.getServiceJob.resolves([job]) engine.docker.getContainer.returns(stubContainer()) engine.pullImageRef = sinon.stub().rejects(new Error('pull failed')) + const snapshot = await engine.restartService(SERVICE_ID, OWNER) + expect(snapshot.status).to.equal(ServiceStatusNumber.Restarting) + await Promise.allSettled([...engine.serviceOpPromises]) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(job.status).to.equal(ServiceStatusNumber.Error) + expect(job.statusText).to.equal('pull failed') + }) + + it('restartService rejects a service whose payment was refunded (never claimed)', async () => { + const engine = makeEngine() + engine.db.getServiceJob.resolves([ + makeJob({ + status: ServiceStatusNumber.Error, + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '0xl', + claimTx: '', + cancelTx: '0xcancel', + cost: 5 + } + }) + ]) + try { await engine.restartService(SERVICE_ID, OWNER) expect.fail('expected restartService to reject') } catch (e: any) { - expect(e.message).to.equal('pull failed') + expect(e.message).to.contain('refunded') } expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) }) it('rejects a concurrent restart/stop while the lock is held, without touching docker or the DB', async () => { @@ -261,4 +301,169 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { await engine.InternalLoop() expect(expired.status).to.equal(ServiceStatusNumber.Expired) }) + + it('restartService rejects when another process holds the DB lease', async () => { + const engine = makeEngine() + engine.db.acquireServiceLock = sinon.stub().resolves(false) // held elsewhere + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('operation in progress') + } + // the in-memory reservation must be rolled back so a later acquire can succeed + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(engine.db.getServiceJob.called).to.equal(false) + }) + + it('InternalLoop skips a pending job whose DB lease is held by another process', async () => { + const engine = makeEngine() + engine.db.getPendingServiceStarts.resolves([ + makeJob({ status: ServiceStatusNumber.PullImage, statusText: 'PullImage' }) + ]) + engine.db.acquireServiceLock = sinon.stub().resolves(false) + engine.processServiceStart = sinon.stub().resolves() + + await engine.InternalLoop() + await flush() + expect(engine.processServiceStart.called).to.equal(false) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + }) + + it('restartService releases the DB lease once the operation settles', async () => { + const engine = makeEngine() + engine.db.getServiceJob.resolves([makeJob()]) + engine.docker.getContainer.returns(stubContainer()) + engine.docker.createNetwork.resolves({ id: 'newnet' }) + const newContainer = stubContainer() + ;(newContainer as any).id = 'newc' + engine.docker.createContainer.resolves(newContainer) + engine.pullImageRef = sinon.stub().resolves() + + await engine.restartService(SERVICE_ID, OWNER) + await Promise.allSettled([...engine.serviceOpPromises]) + expect( + engine.db.releaseServiceLock.calledWith(SERVICE_ID, 'test-holder'), + 'DB lease must be released with the holder id' + ).to.equal(true) + }) + + it('an explicit stop KEEPS the host ports reserved; only expiry releases them', async () => { + // The user-B scenario: A reserves for 1h, starts, stops after 30min. B must NOT be + // able to grab A's port/resources — A can restart anytime until expiresAt. + const PORT = 31555 + const engine = makeEngine() + const job = makeJob({ + endpoints: [ + { containerPort: 8888, hostPort: PORT, url: `http://localhost:${PORT}` } + ] + }) + engine.db.getServiceJob.resolves([job]) + engine.docker.getContainer.returns(stubContainer()) + reserveHostPort(PORT) // as the original start allocation did + + const stopped = await engine.stopService(SERVICE_ID, OWNER) + expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) + // "user B": the allocator must refuse the stopped service's port + let refused = false + try { + await allocateHostPort(PORT, PORT) + } catch { + refused = true + } + expect(refused, "a stopped service's port must stay reserved").to.equal(true) + + // past expiresAt the sweep flips it to Expired and releases the port + job.expiresAt = Date.now() - 1000 + engine.db.getExpiredServiceJobs.resolves([job]) + engine.isInternalLoopRunning = false + await engine.InternalLoop() + expect(job.status).to.equal(ServiceStatusNumber.Expired) + expect(await allocateHostPort(PORT, PORT)).to.equal(PORT) + releaseHostPort(PORT) // tidy up the test's own allocation + }) + + it('the expiry sweep does NOT mark Expired while teardown fails (no resource leak), then retries', async () => { + const engine = makeEngine() + const expired = makeJob({ expiresAt: Date.now() - 1000 }) + engine.db.getExpiredServiceJobs.resolves([expired]) + engine.db.getServiceJob.resolves([expired]) + // teardown fails hard (docker daemon down — NOT a benign 404) + engine.docker.getContainer.returns( + stubContainer({ stop: sinon.stub().rejects(benignError(500)) }) + ) + + await engine.InternalLoop() + // Expired is terminal and never swept again — a failed stop must leave the job + // OUT of Expired (as Error "stop failed") so the container/ports aren't leaked. + expect(expired.status).to.not.equal(ServiceStatusNumber.Expired) + expect(String(expired.statusText)).to.contain('stop failed') + + // docker recovers → the next tick completes the teardown and marks Expired + engine.docker.getContainer.returns(stubContainer()) + engine.isInternalLoopRunning = false + await engine.InternalLoop() + expect(expired.status).to.equal(ServiceStatusNumber.Expired) + }) + + it('the expiry sweep flips a Stopped service past expiresAt to Expired without touching docker', async () => { + const engine = makeEngine() + const stopped = makeJob({ + status: ServiceStatusNumber.Stopped, + statusText: 'Stopped', + containerId: '', + networkId: '', + expiresAt: Date.now() - 1000 + }) + engine.db.getExpiredServiceJobs.resolves([stopped]) + engine.db.getServiceJob.resolves([stopped]) + + await engine.InternalLoop() + + expect(stopped.status).to.equal(ServiceStatusNumber.Expired) + // resources were already released at stop time — no docker teardown must happen + expect(engine.docker.getContainer.called).to.equal(false) + }) + + it('processServiceStart ignores a stale pending snapshot (job already Running again)', async () => { + const engine = makeEngine() + // Snapshot captured while a restart was mid-pull... + const staleSnapshot = makeJob({ + status: ServiceStatusNumber.PullImage, + statusText: 'PullImage', + containerId: '', + networkId: '' + }) + // ...but by the time the loop processes it, the restart finished: the fresh DB row + // is Running with live docker refs. Pre-guard, orphan-recovery would tear that + // container + network down and clobber the status with Error. + engine.db.getServiceJob.resolves([ + makeJob({ + status: ServiceStatusNumber.Running, + containerId: 'c-live', + networkId: 'n-live' + }) + ]) + + await engine.processServiceStart(staleSnapshot) + + expect(engine.docker.getNetwork.called).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + expect(engine.db.updateServiceJob.called).to.equal(false) + }) + + it('a stopped engine refuses new lifecycle operations', async () => { + const engine = makeEngine() + engine.stopped = true + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('stopped') + } + expect(engine.db.getServiceJob.called).to.equal(false) + expect(engine.db.acquireServiceLock.called).to.equal(false) + }) }) From b3f38e18614538a1702bc81e33253e9822cd558b Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 16 Jul 2026 21:37:07 +0300 Subject: [PATCH 05/10] fix more bugs --- docs/services.md | 18 +++++++--- src/components/c2d/compute_engine_docker.ts | 12 ++++--- src/components/core/service/startService.ts | 33 +++++++++++++++++++ src/components/database/sqliteCompute.ts | 15 ++++++++- src/test/unit/compute.test.ts | 4 +-- src/test/unit/service/serviceHandlers.test.ts | 23 ++++++++++++- .../unit/service/serviceJobsDatabase.test.ts | 30 +++++++++++++++++ .../unit/service/serviceRestartRace.test.ts | 30 +++++++++++++++++ 8 files changed, 152 insertions(+), 13 deletions(-) diff --git a/docs/services.md b/docs/services.md index c7b3d0d42..97b5eefcd 100644 --- a/docs/services.md +++ b/docs/services.md @@ -42,8 +42,11 @@ terminal `*Failed` / `Error`). **Handler (synchronous, before responding):** signature check → environment + access-list + `features.services` check → `userData` decrypt (validity check) → duration cap → resource -resolution & availability → cost computed from **server-side** environment pricing → persist the -job as `Starting` (which also reserves its resources) → respond `200` with the `serviceId`. +resolution & availability → cost computed from **server-side** environment pricing → escrow +funds pre-check (fail fast with `400 Insufficient escrow funds` when the consumer's available +escrow visibly can't cover the cost; best-effort — an RPC hiccup skips it and the background +Locking step remains the authoritative check) → persist the job as `Starting` (which also +reserves its resources) → respond `200` with the `serviceId`. **Background pipeline (per the start statuses below):** `Starting (10)` → **locking** `Locking (20)`: escrow `createLock` (+ wait for it to mine) → @@ -62,15 +65,20 @@ locked-then-claimed up front. `Restarting (45)` and responds immediately — the teardown, image re-pull/rebuild and new container happen in the background under the same per-service lock. Poll `serviceStatus` and watch `Restarting` → `PullImage`/`BuildImage` → `Running` (or `Error` with the failure -reason in `statusText`). A service whose start payment was **refunded** (lock cancelled -before it was claimed) cannot be restarted — it was never paid for; start a new service. +reason in `statusText`). A service whose start payment was **never claimed** — the escrow +lock failed outright (e.g. insufficient funds) or was refunded before being claimed — +cannot be restarted: it was never paid for, so restarting it would run the service for +free. Start a new service instead. **The reservation lasts the whole paid window — only `Expired` releases it.** The consumer paid for the resources for a time interval and may use them as they please within it: running the service, stopping it, restarting it. An explicit `SERVICE_STOP` therefore tears down the container/network but **keeps** the resource amounts (cpu/ram/gpu) counted and the host ports reserved — another consumer cannot take them, and a restart resumes on the same -endpoints. Once `expiresAt` passes, the expiry sweep tears down whatever is left, marks the +endpoints. The reservation is tied to **payment**: an `Error`/`Stopped` job whose payment +was never claimed (lock failed or refunded) does not reserve anything — otherwise anyone +could squat a node's GPU for free by starting services against an empty escrow account. +Once `expiresAt` passes, the expiry sweep tears down whatever is left, marks the job `Expired`, and only then releases everything. The sweep refuses to mark `Expired` while teardown fails (e.g. Docker unreachable) — the job stays `Error` and is retried every tick, so a resource release is never silently skipped. diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 3698191f9..d8ba5f689 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -4049,11 +4049,15 @@ export class C2DEngineDocker extends C2DEngine { // Expired. Restarting then would silently extend the service beyond what was paid for. if (job.status === ServiceStatusNumber.Expired || Date.now() >= job.expiresAt) throw new Error('Cannot restart an expired service') - // A refunded start (lock cancelled, never claimed) was never paid for — restarting - // it would run the service for free. The consumer must start a new service instead. - if (job.payment?.cancelTx && !job.payment?.claimTx) + // Only a CLAIMED payment makes a job restartable. Every legitimately restartable + // job (Running, container-died Error, Stopped) has claimTx set — claiming happens + // before the first container start. A job without it was never paid for: either + // the escrow lock failed outright (e.g. insufficient funds — all payment fields + // empty) or the lock was refunded (cancelTx set). Restarting either would run the + // service for free. The consumer must start a new service instead. + if (!job.payment?.claimTx) throw new Error( - 'Cannot restart a service whose payment was refunded — start a new service' + 'Cannot restart a service whose payment was never claimed (unpaid or refunded) — start a new service' ) // Persist Restarting BEFORE returning: status polls flip immediately, and a crash // from here on leaves a pending-status record the boot loop orphan-recovers diff --git a/src/components/core/service/startService.ts b/src/components/core/service/startService.ts index dd2a6737e..585ce69bd 100644 --- a/src/components/core/service/startService.ts +++ b/src/components/core/service/startService.ts @@ -167,6 +167,39 @@ export class ServiceStartHandler extends CommandHandler { ) ) + // 6b. Fail fast when the consumer's escrow visibly can't cover the cost, instead + // of returning a serviceId doomed to fail asynchronously at the Locking step. + // Best-effort UX only: balances can change before the background createLock runs, + // so the authoritative check stays in the pipeline — and an RPC hiccup here must + // not block starts (the pipeline check will catch a genuine shortfall anyway). + try { + const [availableWei, costWei] = await Promise.all([ + engine.escrow.getUserAvailableFunds( + task.payment.chainId, + task.consumerAddress, + task.payment.token + ), + engine.escrow.getPaymentAmountInWei( + cost, + task.payment.chainId, + task.payment.token + ) + ]) + if (BigInt(availableWei.toString()) < BigInt(costWei.toString())) { + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `Insufficient escrow funds for token ${task.payment.token} on chain ` + + `${task.payment.chainId}: available ${availableWei}, required ${costWei} ` + + `wei — deposit and authorize escrow funds before starting the service` + ) + ) + } + } catch (e: any) { + CORE_LOGGER.debug( + `SERVICE_START: escrow funds pre-check skipped (${e.message}) — the background Locking step will verify` + ) + } + const serviceId = generateUniqueID({ owner: task.consumerAddress, environment: task.environment, diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index a0c8368ef..463f5c0d5 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -408,7 +408,20 @@ export class SQLiteCompute implements ComputeDatabaseProvider { DATABASE_LOGGER.error(err.message) reject(err) } else { - resolve(this.mapServiceRows(rows)) + // The reservation is tied to PAYMENT: an Error/Stopped job whose payment was + // never claimed (escrow lock failed — e.g. insufficient funds — or refunded) + // must not hold resources, or anyone could squat a node's GPU for free by + // starting services against an empty escrow account. Mid-pipeline statuses + // keep reserving even without claimTx — they are en route to payment. + // JS-side filter because payment lives in the JSON body, not a SQL column. + resolve( + this.mapServiceRows(rows).filter( + (j) => + (j.status !== ServiceStatusNumber.Error && + j.status !== ServiceStatusNumber.Stopped) || + !!j.payment?.claimTx + ) + ) } }) }) diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 68e5c7424..c91d57838 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -1735,7 +1735,7 @@ describe('service start/restart Docker cleanup on failure', function () { exposedPorts: [80], endpoints: [{ containerPort: 80, hostPort: 30001, url: 'http://localhost:30001' }], resources: [{ id: 'cpu', amount: 1 }], - payment: { chainId: 1, token: '0xtoken' } + payment: { chainId: 1, token: '0xtoken', claimTx: '0xclaim' } } engine.db.getServiceJob = sinon.stub().resolves([existingJob]) engine.docker = { @@ -1774,7 +1774,7 @@ describe('service start/restart Docker cleanup on failure', function () { exposedPorts: [80], endpoints: [{ containerPort: 80, hostPort: 30001, url: 'http://localhost:30001' }], resources: [{ id: 'cpu', amount: 1 }], - payment: { chainId: 1, token: '0xtoken' }, + payment: { chainId: 1, token: '0xtoken', claimTx: '0xclaim' }, dockerCmd: ['old', 'cmd'], dockerEntrypoint: ['/old-entrypoint'], ...overrides diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index 6a20d7294..a851051b0 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -76,7 +76,10 @@ function buildFakes(opts: FakeOpts = {}) { claimLock: sinon.stub().resolves('0xclaim'), cancelExpiredLock: sinon.stub().resolves('0xcancel'), waitForTransaction: sinon.stub().resolves(undefined), - getMinLockTime: sinon.stub().returns(3600) + getMinLockTime: sinon.stub().returns(3600), + // the SERVICE_START handler's fail-fast funds pre-check — plentiful by default + getUserAvailableFunds: sinon.stub().resolves(1_000_000n), + getPaymentAmountInWei: sinon.stub().resolves(10n) } const engine: any = { @@ -444,6 +447,24 @@ describe('Service handlers', () => { expect(res.status.httpStatus).to.equal(400) }) + it('400 with a clear message when the escrow cannot cover the cost (fail fast)', async () => { + const { node, engine } = buildFakes() + engine.escrow.getUserAvailableFunds.resolves(0n) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(400) + expect(String(res.status.error)).to.contain('Insufficient escrow funds') + // no job record may be created for a start that was refused upfront + expect(engine.createServiceJob.called).to.equal(false) + }) + + it('the funds pre-check is best-effort: an RPC failure does not block the start', async () => { + const { node, engine } = buildFakes() + engine.escrow.getUserAvailableFunds.rejects(new Error('rpc down')) + const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + expect(engine.createServiceJob.calledOnce).to.equal(true) + }) + it('403 when services are disabled on the environment', async () => { const { node } = buildFakes({ serviceEnabled: false }) const res = await new ServiceStartHandler(node).handle({ ...baseTask } as any) diff --git a/src/test/unit/service/serviceJobsDatabase.test.ts b/src/test/unit/service/serviceJobsDatabase.test.ts index f1945be00..5ab2e8321 100644 --- a/src/test/unit/service/serviceJobsDatabase.test.ts +++ b/src/test/unit/service/serviceJobsDatabase.test.ts @@ -228,6 +228,30 @@ describe('Service Jobs Database', () => { const restarting = makeServiceJob({ status: ServiceStatusNumber.Restarting }) const error = makeServiceJob({ status: ServiceStatusNumber.Error }) const stopped = makeServiceJob({ status: ServiceStatusNumber.Stopped }) + // never paid: escrow lock failed outright (all payment fields empty) + const unpaidError = makeServiceJob({ + status: ServiceStatusNumber.Error, + payment: { + chainId: 8996, + token: '0x123', + lockTx: '', + claimTx: '', + cancelTx: '', + cost: 5 + } + }) + // never paid: lock was refunded before being claimed + const refundedStopped = makeServiceJob({ + status: ServiceStatusNumber.Stopped, + payment: { + chainId: 8996, + token: '0x123', + lockTx: '0xlock', + claimTx: '', + cancelTx: '0xcancel', + cost: 5 + } + }) const otherCluster = makeServiceJob({ status: ServiceStatusNumber.Running, clusterHash: 'other-cluster' @@ -239,6 +263,8 @@ describe('Service Jobs Database', () => { await db.newServiceJob(restarting) await db.newServiceJob(error) await db.newServiceJob(stopped) + await db.newServiceJob(unpaidError) + await db.newServiceJob(refundedStopped) await db.newServiceJob(otherCluster) const active = await db.getRunningServiceJobs(CLUSTER_HASH) @@ -256,6 +282,10 @@ describe('Service Jobs Database', () => { // an explicitly-stopped service KEEPS its reservation too: the consumer paid for // the whole window and may restart it anytime — only Expired releases resources expect(ids).to.include(stopped.serviceId) + // …but the reservation is tied to PAYMENT: a terminal job whose payment was never + // claimed (lock failed / refunded) must not squat resources + expect(ids).to.not.include(unpaidError.serviceId) + expect(ids).to.not.include(refundedStopped.serviceId) expect(ids).to.not.include(otherCluster.serviceId) }) diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts index 69d0b1fcd..772731f0c 100644 --- a/src/test/unit/service/serviceRestartRace.test.ts +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -182,6 +182,36 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { expect(engine.docker.getContainer.called).to.equal(false) }) + it('restartService rejects a service that was never paid at all (escrow lock failed)', async () => { + // The free-compute vector: start against an empty escrow account → createLock fails + // → Error with ALL payment fields empty → a restart must not run the service anyway. + const engine = makeEngine() + engine.db.getServiceJob.resolves([ + makeJob({ + status: ServiceStatusNumber.Error, + statusText: 'User 0x… does not have enough funds', + payment: { + chainId: 8996, + token: '0xtoken', + lockTx: '', + claimTx: '', + cancelTx: '', + cost: 5 + } + }) + ]) + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('never claimed') + } + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + expect(engine.docker.createNetwork.called).to.equal(false) + }) + it('rejects a concurrent restart/stop while the lock is held, without touching docker or the DB', async () => { const engine = makeEngine() engine.serviceOpsInFlight.add(SERVICE_ID) From 40cc7414a8b1e019e0f6d74932f947a19759f21e Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 16 Jul 2026 21:59:22 +0300 Subject: [PATCH 06/10] fix review comments --- src/components/c2d/compute_engine_base.ts | 16 +- src/components/c2d/compute_engine_docker.ts | 77 ++++- src/components/core/service/extendService.ts | 286 ++++++++++-------- src/components/database/sqliteCompute.ts | 9 +- src/test/integration/services.test.ts | 11 +- src/test/unit/compute.test.ts | 44 ++- src/test/unit/service/serviceHandlers.test.ts | 5 +- .../unit/service/serviceRestartRace.test.ts | 64 +++- 8 files changed, 355 insertions(+), 157 deletions(-) diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index a3a8c4d7a..54b6c0fd0 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -110,11 +110,25 @@ export abstract class C2DEngine { // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await public async processServiceStart(job: ServiceJob): Promise {} + // onlyIfExpired: expiry-sweep mode — re-validate expiresAt on the fresh row under the + // lifecycle lock and skip the teardown when the service was extended in the meantime. // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await - public async stopService(serviceId: string, owner: string): Promise { + public async stopService( + serviceId: string, + owner: string, + onlyIfExpired?: boolean + ): Promise { return null } + // Runs fn serialized with the engine's per-service lifecycle operations (start + // pipeline, restart, stop, expiry sweep). Engines without a lock implementation run + // fn directly; C2DEngineDocker overrides this with its lifecycle lock + DB lease. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public async runExclusive(serviceId: string, fn: () => Promise): Promise { + return await fn() + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars, require-await public async restartService( serviceId: string, diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index d8ba5f689..1811ca1a0 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -1836,6 +1836,14 @@ export class C2DEngineDocker extends C2DEngine { return this.releaseServiceLifecycleLock(svc.serviceId) }) this.serviceOpPromises.add(startPromise) + // processServiceStart persists every outcome, but its up-front DB re-read can + // still reject — consume it so the unawaited promise can't surface as an + // unhandled rejection (stop()'s allSettled drain never observes rejections). + startPromise.catch((e) => + CORE_LOGGER.error( + `processServiceStart ${svc.serviceId} failed unexpectedly: ${e.message}` + ) + ) } // Service-on-Demand expiry: stop services whose paid window has elapsed. @@ -1846,7 +1854,10 @@ export class C2DEngineDocker extends C2DEngine { CORE_LOGGER.info(`Service ${svc.serviceId} expired — stopping`) let stopped: ServiceJob | null try { - stopped = await this.stopService(svc.serviceId, svc.owner) + // onlyIfExpired: doStopService re-checks expiresAt on the FRESH row under the + // lifecycle lock — a SERVICE_EXTEND that landed after our expiredServices + // snapshot must not have its service torn down. + stopped = await this.stopService(svc.serviceId, svc.owner, true) } catch (e: any) { // Typically the lifecycle lock (a restart/stop already in flight). Marking // Expired without teardown would leak the container/ports, so defer the whole @@ -1856,6 +1867,14 @@ export class C2DEngineDocker extends C2DEngine { ) continue } + // Extended mid-sweep (expiresAt is in the future again on the fresh row) — not + // expired anymore, regardless of status. Leave it alone. + if (stopped && Date.now() < stopped.expiresAt) { + CORE_LOGGER.debug( + `Service ${svc.serviceId} was extended after the expiry snapshot — skipping expiry` + ) + continue + } // Flip to Expired ONLY once teardown actually completed. Expired is terminal — // it is never swept again — so marking it while the stop failed mid-way (the // job comes back as Error "stop failed: …" with its container possibly still @@ -3843,9 +3862,11 @@ export class C2DEngineDocker extends C2DEngine { // A fresh DB lease means ANOTHER process is mid-operation on this service (its // teardown makes the container look dead); our in-memory serviceOpsInFlight can't // see that. Skip — if the container is genuinely dead, the next tick catches it. + // Fail CLOSED: when the lease state can't be read, assume locked and skip this + // tick rather than risk flipping a mid-restart service to Error. const lockedElsewhere = await this.db .isServiceLocked(job.serviceId, SERVICE_LOCK_STALE_MS) - .catch(() => false) + .catch(() => true) if (lockedElsewhere) { CORE_LOGGER.debug( `markServiceFailed ${job.serviceId}: skipped — a lifecycle lease is held elsewhere` @@ -3889,12 +3910,19 @@ export class C2DEngineDocker extends C2DEngine { CORE_LOGGER.debug( `service lock ${serviceId}: DB lease held by another process — not acquired` ) - } else { + return false + } + // Re-check stopped: the engine may have been stopped (and drained) while the async + // DB acquire was in flight — beginning work now would race the replacement engine. + if (this.stopped) { CORE_LOGGER.debug( - `service lock ${serviceId}: acquired by ${this.serviceLockHolderId}` + `service lock ${serviceId}: engine stopped during acquire — releasing` ) + await this.releaseServiceLifecycleLock(serviceId) + return false } - return acquired + CORE_LOGGER.debug(`service lock ${serviceId}: acquired by ${this.serviceLockHolderId}`) + return true } // Takes the per-service lifecycle lock or throws: a stop must not run while the loop's @@ -3929,11 +3957,29 @@ export class C2DEngineDocker extends C2DEngine { public override async stopService( serviceId: string, - owner: string + owner: string, + onlyIfExpired: boolean = false ): Promise { await this.acquireServiceLifecycleLock(serviceId) // Tracked like loop-launched starts so engine stop() drains an in-flight stop too. - const op = this.doStopService(serviceId, owner).finally(() => { + const op = this.doStopService(serviceId, owner, onlyIfExpired).finally(() => { + this.serviceOpPromises.delete(op) + return this.releaseServiceLifecycleLock(serviceId) + }) + this.serviceOpPromises.add(op) + return await op + } + + // Runs fn under the per-service lifecycle lock (in-memory + cross-process DB lease), + // serializing it with the start pipeline, restart, stop and the expiry sweep. Used by + // handlers whose read-mutate-write flows (e.g. SERVICE_EXTEND) must not interleave + // with a concurrent teardown — throws "operation in progress" when the lock is busy. + public override async runExclusive( + serviceId: string, + fn: () => Promise + ): Promise { + await this.acquireServiceLifecycleLock(serviceId) + const op = fn().finally(() => { this.serviceOpPromises.delete(op) return this.releaseServiceLifecycleLock(serviceId) }) @@ -3945,13 +3991,25 @@ export class C2DEngineDocker extends C2DEngine { // stopService above). private async doStopService( serviceId: string, - owner: string + owner: string, + onlyIfExpired: boolean = false ): Promise { const [job] = await this.db.getServiceJob(serviceId, owner) if (!job) { CORE_LOGGER.debug(`stopService ${serviceId}: no job found for owner ${owner}`) return null } + // Expiry-sweep mode: re-validate expiry on the FRESH row now that we hold the lock. + // The sweep's expiredServices snapshot predates the lock, so a SERVICE_EXTEND may + // have pushed expiresAt into the future since — tearing down then would destroy a + // service the consumer just paid to prolong. The caller sees the untouched job and + // skips (its own expiresAt check). + if (onlyIfExpired && Date.now() < job.expiresAt) { + CORE_LOGGER.debug( + `stopService ${serviceId}: expiry teardown requested but the job was extended (expiresAt in the future) — skipping` + ) + return job + } if ( job.status === ServiceStatusNumber.Stopped || job.status === ServiceStatusNumber.Expired @@ -4037,6 +4095,9 @@ export class C2DEngineDocker extends C2DEngine { ;[job] = await this.db.getServiceJob(serviceId, owner) if (!job) { CORE_LOGGER.debug(`restartService ${serviceId}: no job found for owner ${owner}`) + // early return bypasses both the catch below and the background op's finally — + // the lock must be handed back explicitly or it leaks forever + await this.releaseServiceLifecycleLock(serviceId) return null } CORE_LOGGER.debug( diff --git a/src/components/core/service/extendService.ts b/src/components/core/service/extendService.ts index 63114e6c3..d4f824a3d 100644 --- a/src/components/core/service/extendService.ts +++ b/src/components/core/service/extendService.ts @@ -93,138 +93,174 @@ export class ServiceExtendHandler extends CommandHandler { if (!accessGranted) return { stream: null, status: { httpStatus: 403, error: 'Access denied' } } - // State check — only Starting or Running can be extended - if ( - job.status !== ServiceStatusNumber.Starting && - job.status !== ServiceStatusNumber.Running - ) - return buildInvalidParametersResponse( - buildInvalidRequestMessage( - `Cannot extend a service in state "${job.statusText}". Only Starting or Running services can be extended.` - ) - ) + // Everything from the state check to the final write runs under the per-service + // lifecycle lock: extend is a read-mutate-write of the job row, and without the lock + // its final updateServiceJob could overwrite a concurrent stop/restart/Expired state + // (and the expiry sweep could tear the service down between our payment and write). + // The lock is taken BEFORE any escrow operation, so a busy lock costs nothing. + try { + return await engine.runExclusive( + task.serviceId, + async (): Promise => { + // Re-read under the lock — every other mutator holds the same lock, so this + // snapshot cannot go stale before our write. + const [freshJob] = await engine.db.getServiceJob( + task.serviceId, + task.consumerAddress + ) + if (!freshJob) + return buildInvalidParametersResponse( + buildInvalidRequestMessage('Service job not found: ' + task.serviceId) + ) - // Extension must not push total beyond maxDurationSeconds - const sod = engine.getC2DConfig().connection?.serviceOnDemand - const maxDuration = sod?.maxDurationSeconds ?? 86400 - const remainingSeconds = Math.max(0, Math.floor((job.expiresAt - Date.now()) / 1000)) - const newTotalDuration = remainingSeconds + task.additionalDuration - if (newTotalDuration > maxDuration) - return buildInvalidParametersResponse( - buildInvalidRequestMessage( - `Extension would result in ${newTotalDuration}s remaining, exceeding maximum ${maxDuration}s` - ) - ) + // State check — only Starting or Running can be extended + if ( + freshJob.status !== ServiceStatusNumber.Starting && + freshJob.status !== ServiceStatusNumber.Running + ) + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `Cannot extend a service in state "${freshJob.statusText}". Only Starting or Running services can be extended.` + ) + ) - // Cost — same price formula as the start, priced off the env the service runs on. - // No fallback: pricing must use runEnv (resolved above); calculateResourcesCost returns - // null if that env has no pricing for the token, handled by the check below. - const costExtend = engine.calculateResourcesCost( - job.resources.map((r) => ({ id: r.id, amount: r.amount })), - runEnv, - task.payment.chainId, - task.payment.token, - task.additionalDuration - ) - if (costExtend === null) - return buildInvalidParametersResponse( - buildInvalidRequestMessage( - `No pricing configured for token ${task.payment.token} on chain ${task.payment.chainId}` - ) - ) + // Extension must not push total beyond maxDurationSeconds + const sod = engine.getC2DConfig().connection?.serviceOnDemand + const maxDuration = sod?.maxDurationSeconds ?? 86400 + const remainingSeconds = Math.max( + 0, + Math.floor((freshJob.expiresAt - Date.now()) / 1000) + ) + const newTotalDuration = remainingSeconds + task.additionalDuration + if (newTotalDuration > maxDuration) + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `Extension would result in ${newTotalDuration}s remaining, exceeding maximum ${maxDuration}s` + ) + ) - // Escrow lock + immediate claim - let lockTx: string | null - try { - lockTx = await engine.escrow.createLock( - task.payment.chainId, - task.serviceId, - task.payment.token, - task.consumerAddress, - costExtend, - engine.escrow.getMinLockTime(task.additionalDuration) - ) - } catch (e: any) { - CORE_LOGGER.error(`Service extend createLock failed: ${e.message}`) - return { stream: null, status: { httpStatus: 402, error: e.message } } - } - if (!lockTx) - return { - stream: null, - status: { httpStatus: 402, error: 'Escrow lock failed for extend' } - } + // Cost — same price formula as the start, priced off the env the service runs + // on. No fallback: pricing must use runEnv (resolved above); + // calculateResourcesCost returns null if that env has no pricing for the token. + const costExtend = engine.calculateResourcesCost( + freshJob.resources.map((r) => ({ id: r.id, amount: r.amount })), + runEnv, + task.payment.chainId, + task.payment.token, + task.additionalDuration + ) + if (costExtend === null) + return buildInvalidParametersResponse( + buildInvalidRequestMessage( + `No pricing configured for token ${task.payment.token} on chain ${task.payment.chainId}` + ) + ) - // Wait for the lock tx to be mined before claiming (same-signer back-to-back txs). - try { - await engine.escrow.waitForTransaction(task.payment.chainId, lockTx) - } catch (e: any) { - CORE_LOGGER.error(`Service extend lock not confirmed: ${e.message}`) - await engine.escrow - .cancelExpiredLock( - task.payment.chainId, - task.serviceId, - task.payment.token, - task.consumerAddress - ) - .catch((err) => CORE_LOGGER.error(`cancelExpiredLock failed: ${err.message}`)) - return { - stream: null, - status: { httpStatus: 402, error: 'Escrow lock not confirmed — lock cancelled' } - } - } + // Escrow lock + immediate claim + let lockTx: string | null + try { + lockTx = await engine.escrow.createLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress, + costExtend, + engine.escrow.getMinLockTime(task.additionalDuration) + ) + } catch (e: any) { + CORE_LOGGER.error(`Service extend createLock failed: ${e.message}`) + return { stream: null, status: { httpStatus: 402, error: e.message } } + } + if (!lockTx) + return { + stream: null, + status: { httpStatus: 402, error: 'Escrow lock failed for extend' } + } - let claimTx: string | null - try { - claimTx = await engine.escrow.claimLock( - task.payment.chainId, - task.serviceId, - task.payment.token, - task.consumerAddress, - costExtend, - `service-extend:${task.serviceId}` - ) - } catch (e: any) { - claimTx = null - CORE_LOGGER.error(`Service extend claimLock failed: ${e.message}`) - } - if (!claimTx) { - await engine.escrow - .cancelExpiredLock( - task.payment.chainId, - task.serviceId, - task.payment.token, - task.consumerAddress - ) - .catch((e) => CORE_LOGGER.error(`cancelExpiredLock failed: ${e.message}`)) - return { - stream: null, - status: { httpStatus: 402, error: 'Escrow claim failed — lock cancelled' } - } - } + // Wait for the lock tx to be mined before claiming (same-signer back-to-back txs). + try { + await engine.escrow.waitForTransaction(task.payment.chainId, lockTx) + } catch (e: any) { + CORE_LOGGER.error(`Service extend lock not confirmed: ${e.message}`) + await engine.escrow + .cancelExpiredLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress + ) + .catch((err) => + CORE_LOGGER.error(`cancelExpiredLock failed: ${err.message}`) + ) + return { + stream: null, + status: { + httpStatus: 402, + error: 'Escrow lock not confirmed — lock cancelled' + } + } + } - // Payment successful — push expiresAt forward and record extension payment - job.expiresAt += task.additionalDuration * 1000 - job.duration += task.additionalDuration - job.extendPayments = [ - ...(job.extendPayments ?? []), - { - chainId: task.payment.chainId, - token: task.payment.token, - lockTx, - claimTx, - cancelTx: '', - cost: costExtend - } - ] - await engine.db.updateServiceJob(job) + let claimTx: string | null + try { + claimTx = await engine.escrow.claimLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress, + costExtend, + `service-extend:${task.serviceId}` + ) + } catch (e: any) { + claimTx = null + CORE_LOGGER.error(`Service extend claimLock failed: ${e.message}`) + } + if (!claimTx) { + await engine.escrow + .cancelExpiredLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress + ) + .catch((e) => CORE_LOGGER.error(`cancelExpiredLock failed: ${e.message}`)) + return { + stream: null, + status: { httpStatus: 402, error: 'Escrow claim failed — lock cancelled' } + } + } - CORE_LOGGER.logMessage( - `Service ${task.serviceId} extended by ${task.additionalDuration}s, new expiresAt: ${job.expiresAt}`, - true - ) - return { - stream: Readable.from(JSON.stringify([toPublicServiceJob(job)])), - status: { httpStatus: 200 } + // Payment successful — push expiresAt forward and record extension payment + freshJob.expiresAt += task.additionalDuration * 1000 + freshJob.duration += task.additionalDuration + freshJob.extendPayments = [ + ...(freshJob.extendPayments ?? []), + { + chainId: task.payment.chainId, + token: task.payment.token, + lockTx, + claimTx, + cancelTx: '', + cost: costExtend + } + ] + await engine.db.updateServiceJob(freshJob) + + CORE_LOGGER.logMessage( + `Service ${task.serviceId} extended by ${task.additionalDuration}s, new expiresAt: ${freshJob.expiresAt}`, + true + ) + return { + stream: Readable.from(JSON.stringify([toPublicServiceJob(freshJob)])), + status: { httpStatus: 200 } + } + } + ) + } catch (error: any) { + // Lifecycle lock busy (a stop/restart/expiry owns the service) or engine stopped — + // nothing was charged yet, the client simply retries shortly. + CORE_LOGGER.error(`ServiceExtend ${task.serviceId} rejected: ${error.message}`) + return { stream: null, status: { httpStatus: 400, error: error.message } } } } } diff --git a/src/components/database/sqliteCompute.ts b/src/components/database/sqliteCompute.ts index 463f5c0d5..59b5a477b 100644 --- a/src/components/database/sqliteCompute.ts +++ b/src/components/database/sqliteCompute.ts @@ -431,11 +431,16 @@ export class SQLiteCompute implements ComputeDatabaseProvider { // Running, Error AND Stopped all still hold their paid reservation (see activeStatuses // above), so all three must be swept once past expiresAt: the sweep is the ONLY place // the reservation is released. Without it an abandoned Error or Stopped service would - // keep its resources/ports forever and still read as restartable. + // keep its resources/ports forever and still read as restartable. Stopping is swept + // too: a process crash mid-stop persists that status and nothing else recovers it + // (it is not a pending-start status), so it would otherwise be stuck reserving + // resources forever — the sweep's doStopService handles a Stopping row like any + // other teardown (benign 404s for whatever the crashed stop already removed). const expirableStatuses = [ ServiceStatusNumber.Running, ServiceStatusNumber.Error, - ServiceStatusNumber.Stopped + ServiceStatusNumber.Stopped, + ServiceStatusNumber.Stopping ] const placeholders = expirableStatuses.map(() => '?').join(',') const params: Array = [...expirableStatuses, Date.now()] diff --git a/src/test/integration/services.test.ts b/src/test/integration/services.test.ts index 449f2d39b..409081d48 100644 --- a/src/test/integration/services.test.ts +++ b/src/test/integration/services.test.ts @@ -813,13 +813,20 @@ describe('********** Service on Demand', () => { } const restartPromise = new ServiceRestartHandler(oceanNode).handle(task) - // wait until the restart is actually mid-pull (deterministic, not sleep-based) + // wait until the restart is actually mid-pull (deterministic, not sleep-based) — + // and ASSERT it got there: silently timing out would run the concurrent-command + // checks against a restart in the wrong phase, proving nothing. const deadline = Date.now() + 10_000 + let sawPullImage = false while (Date.now() < deadline) { const j = await getServiceJob(serviceId) - if (j?.status === ServiceStatusNumber.PullImage) break + if (j?.status === ServiceStatusNumber.PullImage) { + sawPullImage = true + break + } await sleep(250) } + assert(sawPullImage, 'restart must reach PullImage within the wait window') // concurrent lifecycle operations must be rejected while the restart holds the lock const stopSig = await signFor(consumerAccount, PROTOCOL_COMMANDS.SERVICE_STOP) diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index c91d57838..90412598a 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -1639,8 +1639,14 @@ describe('service start/restart Docker cleanup on failure', function () { // allocator must refuse to hand the port out again. expect(job.endpoints.length).to.be.greaterThan(0) const reservedPort = job.endpoints[0].hostPort - await expectRejects(allocateHostPort(reservedPort, reservedPort), 'No free host port') - releaseHostPort(reservedPort) // don't leak the reservation into other tests + try { + await expectRejects( + allocateHostPort(reservedPort, reservedPort), + 'No free host port' + ) + } finally { + releaseHostPort(reservedPort) // don't leak the reservation into other tests + } }) it('processServiceStart refunds (cancelLock) and marks PullImageFailed when the image pull fails', async function () { @@ -1753,6 +1759,12 @@ describe('service start/restart Docker cleanup on failure', function () { expect(network.remove.called, 'network.remove should be called').to.equal(true) expect(existingJob.status).to.equal(ServiceStatusNumber.Error) expect(existingJob.statusText).to.contain('createContainer failed') + // the Error outcome must be PERSISTED — status polls read the DB, not memory + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ serviceId: 'svc-3', status: ServiceStatusNumber.Error }) + ) + ).to.equal(true) }) function makeRunningJobWithCmd(overrides: any = {}) { @@ -1805,6 +1817,16 @@ describe('service start/restart Docker cleanup on failure', function () { expect(createArgs.Entrypoint).to.deep.equal(['/new-entrypoint']) expect(existingJob.dockerCmd).to.deep.equal(['new', 'cmd']) expect(existingJob.dockerEntrypoint).to.deep.equal(['/new-entrypoint']) + // the override must be PERSISTED so future restarts reuse it + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ + status: ServiceStatusNumber.Running, + dockerCmd: ['new', 'cmd'], + dockerEntrypoint: ['/new-entrypoint'] + }) + ) + ).to.equal(true) }) it('restartService reuses the stored dockerCmd/dockerEntrypoint when none are supplied', async function () { @@ -1825,6 +1847,15 @@ describe('service start/restart Docker cleanup on failure', function () { expect(createArgs.Entrypoint).to.deep.equal(['/old-entrypoint']) expect(existingJob.dockerCmd).to.deep.equal(['old', 'cmd']) expect(existingJob.dockerEntrypoint).to.deep.equal(['/old-entrypoint']) + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ + status: ServiceStatusNumber.Running, + dockerCmd: ['old', 'cmd'], + dockerEntrypoint: ['/old-entrypoint'] + }) + ) + ).to.equal(true) }) it('restartService clears dockerCmd/dockerEntrypoint when explicitly given an empty array', async function () { @@ -1845,5 +1876,14 @@ describe('service start/restart Docker cleanup on failure', function () { expect(createArgs.Entrypoint).to.equal(undefined) expect(existingJob.dockerCmd).to.deep.equal([]) expect(existingJob.dockerEntrypoint).to.deep.equal([]) + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ + status: ServiceStatusNumber.Running, + dockerCmd: [], + dockerEntrypoint: [] + }) + ) + ).to.equal(true) }) }) diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index a851051b0..ad4fb6c93 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -142,7 +142,10 @@ function buildFakes(opts: FakeOpts = {}) { opts.streamableLogs === undefined ? Readable.from(['hello logs']) : opts.streamableLogs - ) + ), + // handlers (SERVICE_EXTEND) serialize read-mutate-write flows through this; the + // fake just runs the callback (the real engine wraps it in the lifecycle lock) + runExclusive: sinon.stub().callsFake((_id: string, fn: () => Promise) => fn()) } const engines: any = { diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts index 772731f0c..7db55c2f9 100644 --- a/src/test/unit/service/serviceRestartRace.test.ts +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -154,6 +154,17 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) expect(job.status).to.equal(ServiceStatusNumber.Error) expect(job.statusText).to.equal('pull failed') + // the failure must be PERSISTED, not just mutated in memory — status polls read the DB + expect( + engine.db.updateServiceJob.calledWith( + sinon.match({ + serviceId: SERVICE_ID, + status: ServiceStatusNumber.Error, + statusText: 'pull failed' + }) + ), + 'Error status must be written to the DB' + ).to.equal(true) }) it('restartService rejects a service whose payment was refunded (never claimed)', async () => { @@ -392,26 +403,47 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { engine.db.getServiceJob.resolves([job]) engine.docker.getContainer.returns(stubContainer()) reserveHostPort(PORT) // as the original start allocation did - - const stopped = await engine.stopService(SERVICE_ID, OWNER) - expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) - // "user B": the allocator must refuse the stopped service's port - let refused = false try { - await allocateHostPort(PORT, PORT) - } catch { - refused = true + const stopped = await engine.stopService(SERVICE_ID, OWNER) + expect(stopped.status).to.equal(ServiceStatusNumber.Stopped) + // "user B": the allocator must refuse the stopped service's port + let refused = false + try { + await allocateHostPort(PORT, PORT) + } catch { + refused = true + } + expect(refused, "a stopped service's port must stay reserved").to.equal(true) + + // past expiresAt the sweep flips it to Expired and releases the port + job.expiresAt = Date.now() - 1000 + engine.db.getExpiredServiceJobs.resolves([job]) + engine.isInternalLoopRunning = false + await engine.InternalLoop() + expect(job.status).to.equal(ServiceStatusNumber.Expired) + expect(await allocateHostPort(PORT, PORT)).to.equal(PORT) + } finally { + // idempotent — guarantees a failing assertion can't leak the reservation into + // other tests (the allocator set is module-level) + releaseHostPort(PORT) } - expect(refused, "a stopped service's port must stay reserved").to.equal(true) + }) + + it('the expiry sweep skips a service that was extended after the expiry snapshot', async () => { + const engine = makeEngine() + // the sweep's snapshot says "expired", but the FRESH row (re-read under the lock by + // doStopService) was extended in the meantime — teardown must not happen + const staleSnapshot = makeJob({ expiresAt: Date.now() - 1000 }) + const extended = makeJob({ expiresAt: Date.now() + 3600_000 }) + engine.db.getExpiredServiceJobs.resolves([staleSnapshot]) + engine.db.getServiceJob.resolves([extended]) + engine.docker.getContainer.returns(stubContainer()) - // past expiresAt the sweep flips it to Expired and releases the port - job.expiresAt = Date.now() - 1000 - engine.db.getExpiredServiceJobs.resolves([job]) - engine.isInternalLoopRunning = false await engine.InternalLoop() - expect(job.status).to.equal(ServiceStatusNumber.Expired) - expect(await allocateHostPort(PORT, PORT)).to.equal(PORT) - releaseHostPort(PORT) // tidy up the test's own allocation + + expect(extended.status).to.equal(ServiceStatusNumber.Running) + expect(engine.docker.getContainer.called, 'no teardown may happen').to.equal(false) + expect(engine.db.updateServiceJob.called, 'no state may be written').to.equal(false) }) it('the expiry sweep does NOT mark Expired while teardown fails (no resource leak), then retries', async () => { From ff48b73d486447dc1ef14eb8120d24b83cd3f8c7 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 16 Jul 2026 23:16:35 +0300 Subject: [PATCH 07/10] fix lint --- src/components/c2d/compute_engine_docker.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 1811ca1a0..7a67508bc 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -3921,7 +3921,9 @@ export class C2DEngineDocker extends C2DEngine { await this.releaseServiceLifecycleLock(serviceId) return false } - CORE_LOGGER.debug(`service lock ${serviceId}: acquired by ${this.serviceLockHolderId}`) + CORE_LOGGER.debug( + `service lock ${serviceId}: acquired by ${this.serviceLockHolderId}` + ) return true } From beb981d4efde61b3b9c37c847ff6f91fef339911 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Thu, 16 Jul 2026 23:43:37 +0300 Subject: [PATCH 08/10] fixes --- src/components/c2d/compute_engine_docker.ts | 55 +++++++------ src/components/core/service/extendService.ts | 78 +++++++++++++++---- src/test/unit/service/serviceHandlers.test.ts | 51 +++++++++++- .../unit/service/serviceJobsDatabase.test.ts | 9 ++- .../unit/service/serviceRestartRace.test.ts | 5 ++ 5 files changed, 156 insertions(+), 42 deletions(-) diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index 7a67508bc..c3a437976 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -3849,34 +3849,36 @@ export class C2DEngineDocker extends C2DEngine { // restartService() reuses them (and does its own best-effort teardown of the dead // container/network first). private async markServiceFailed(job: ServiceJob, reason: string): Promise { - const [fresh] = await this.db.getServiceJob(job.serviceId, job.owner) - if (!fresh || fresh.status !== ServiceStatusNumber.Running) { - // already moved on (stopped/restarted/expired) by another path — don't clobber it - return - } - if (fresh.containerId !== job.containerId) { - // the container we observed dead is no longer the job's container (a restart - // replaced it since our snapshot) — the current one is not known to be dead - return - } - // A fresh DB lease means ANOTHER process is mid-operation on this service (its - // teardown makes the container look dead); our in-memory serviceOpsInFlight can't - // see that. Skip — if the container is genuinely dead, the next tick catches it. - // Fail CLOSED: when the lease state can't be read, assume locked and skip this - // tick rather than risk flipping a mid-restart service to Error. - const lockedElsewhere = await this.db - .isServiceLocked(job.serviceId, SERVICE_LOCK_STALE_MS) - .catch(() => true) - if (lockedElsewhere) { + // Take the SAME lifecycle lease every start/stop/restart holds, so the Error write + // is serialized with them — a check-then-write here could otherwise overwrite a + // Restarting state persisted between our lease check and our update. Failing to + // acquire (op in flight locally or in another process) fails CLOSED: skip this + // tick; a genuinely dead container is re-detected on the next one. + if (!(await this.tryAcquireServiceLifecycleLock(job.serviceId))) { CORE_LOGGER.debug( - `markServiceFailed ${job.serviceId}: skipped — a lifecycle lease is held elsewhere` + `markServiceFailed ${job.serviceId}: skipped — could not take the lifecycle lease` ) return } - fresh.status = ServiceStatusNumber.Error - fresh.statusText = `service container exited unexpectedly: ${reason}` - await this.db.updateServiceJob(fresh) - CORE_LOGGER.error(`Service ${job.serviceId} container died — ${reason}`) + try { + // Re-read UNDER the lease: this snapshot cannot go stale before our write. + const [fresh] = await this.db.getServiceJob(job.serviceId, job.owner) + if (!fresh || fresh.status !== ServiceStatusNumber.Running) { + // already moved on (stopped/restarted/expired) by another path — don't clobber it + return + } + if (fresh.containerId !== job.containerId) { + // the container we observed dead is no longer the job's container (a restart + // replaced it since our snapshot) — the current one is not known to be dead + return + } + fresh.status = ServiceStatusNumber.Error + fresh.statusText = `service container exited unexpectedly: ${reason}` + await this.db.updateServiceJob(fresh) + CORE_LOGGER.error(`Service ${job.serviceId} container died — ${reason}`) + } finally { + await this.releaseServiceLifecycleLock(job.serviceId) + } } // Tries to take the per-service lifecycle lock: the in-memory set serializes callers @@ -4030,7 +4032,6 @@ export class C2DEngineDocker extends C2DEngine { job.status = ServiceStatusNumber.Stopping job.statusText = ServiceStatusText[ServiceStatusNumber.Stopping] await this.db.updateServiceJob(job) - this.releaseCpus(serviceId) // Benign "already gone" errors count as success. Any other error means teardown // genuinely failed — record it and keep the job OUT of Stopped so the persisted @@ -4069,6 +4070,10 @@ export class C2DEngineDocker extends C2DEngine { // mistakes them for live resources (a restart re-creates and re-persists both). job.containerId = '' job.networkId = '' + // Release the CPU pinning only now that the container is confirmed gone: on a + // failed teardown the old container may still be running on those cores, and + // handing them to another job would double-pin them. + this.releaseCpus(serviceId) CORE_LOGGER.debug(`stopService ${serviceId}: stopped, container + network removed`) } await this.db.updateServiceJob(job) diff --git a/src/components/core/service/extendService.ts b/src/components/core/service/extendService.ts index d4f824a3d..33ee1aef4 100644 --- a/src/components/core/service/extendService.ts +++ b/src/components/core/service/extendService.ts @@ -156,6 +156,43 @@ export class ServiceExtendHandler extends CommandHandler { ) ) + // An extendPayments entry with a lockTx but neither claimTx nor cancelTx is an + // UNRESOLVED intent from a previous crash (see below — the intent is persisted + // before claiming). Resolve it before charging again: try to cancel (refund) + // the old lock; if that fails the lock was most likely already claimed and a + // human must reconcile — reject rather than risk a double charge. + const unresolved = (freshJob.extendPayments ?? []).find( + (p) => p.lockTx && !p.claimTx && !p.cancelTx + ) + if (unresolved) { + try { + const cancelTx = await engine.escrow.cancelExpiredLock( + unresolved.chainId, + task.serviceId, + unresolved.token, + task.consumerAddress + ) + if (!cancelTx) throw new Error('cancelExpiredLock returned no tx') + unresolved.cancelTx = cancelTx + await engine.db.updateServiceJob(freshJob) + CORE_LOGGER.logMessage( + `Service ${task.serviceId}: refunded unresolved extension lock ${unresolved.lockTx} (${cancelTx})`, + true + ) + } catch (e: any) { + CORE_LOGGER.error( + `Service ${task.serviceId}: unresolved extension intent (lock ${unresolved.lockTx}) could not be refunded: ${e.message}` + ) + return { + stream: null, + status: { + httpStatus: 409, + error: `A previous extension of this service is unresolved (lock ${unresolved.lockTx}) and could not be auto-refunded — retry later or contact the node operator` + } + } + } + } + // Escrow lock + immediate claim let lockTx: string | null try { @@ -201,6 +238,21 @@ export class ServiceExtendHandler extends CommandHandler { } } + // Persist the extension INTENT (lockTx recorded, claim pending) BEFORE + // claiming: a crash between claim and the final write is then auditable — the + // consumer's money can never be taken without a durable record of why — and a + // retry finds the intent (unresolved branch above) instead of charging twice. + const intent = { + chainId: task.payment.chainId, + token: task.payment.token, + lockTx, + claimTx: '', + cancelTx: '', + cost: costExtend + } + freshJob.extendPayments = [...(freshJob.extendPayments ?? []), intent] + await engine.db.updateServiceJob(freshJob) + let claimTx: string | null try { claimTx = await engine.escrow.claimLock( @@ -216,34 +268,32 @@ export class ServiceExtendHandler extends CommandHandler { CORE_LOGGER.error(`Service extend claimLock failed: ${e.message}`) } if (!claimTx) { - await engine.escrow + const cancelTx = await engine.escrow .cancelExpiredLock( task.payment.chainId, task.serviceId, task.payment.token, task.consumerAddress ) - .catch((e) => CORE_LOGGER.error(`cancelExpiredLock failed: ${e.message}`)) + .catch((e): string | null => { + CORE_LOGGER.error(`cancelExpiredLock failed: ${e.message}`) + return null + }) + // close the intent (refunded) — or leave it unresolved for the next attempt + if (cancelTx) { + intent.cancelTx = cancelTx + await engine.db.updateServiceJob(freshJob) + } return { stream: null, status: { httpStatus: 402, error: 'Escrow claim failed — lock cancelled' } } } - // Payment successful — push expiresAt forward and record extension payment + // Payment successful — finalize the intent and push expiresAt forward + intent.claimTx = claimTx freshJob.expiresAt += task.additionalDuration * 1000 freshJob.duration += task.additionalDuration - freshJob.extendPayments = [ - ...(freshJob.extendPayments ?? []), - { - chainId: task.payment.chainId, - token: task.payment.token, - lockTx, - claimTx, - cancelTx: '', - cost: costExtend - } - ] await engine.db.updateServiceJob(freshJob) CORE_LOGGER.logMessage( diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index ad4fb6c93..4cf67bc3d 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -410,12 +410,61 @@ describe('Service handlers', () => { expect(res.status.httpStatus).to.equal(200) expect(escrow.createLock.calledOnce).to.equal(true) expect(escrow.claimLock.calledOnce).to.equal(true) - expect(engine.db.updateServiceJob.calledOnce).to.equal(true) + // two writes: the durable intent (before claim) + the finalized extension + expect(engine.db.updateServiceJob.calledTwice).to.equal(true) const out = await body(res) expect(out[0].expiresAt).to.equal(before + 3600 * 1000) expect(out[0].extendPayments).to.have.length(1) + expect(out[0].extendPayments[0].claimTx).to.equal('0xclaim') expect(out[0]).to.not.have.property('userData') }) + + it('auto-refunds an unresolved extension intent from a previous crash, then proceeds', async () => { + const job = makeJob({ + extendPayments: [ + // lockTx set, neither claimTx nor cancelTx — a crash between claim and write + { + chainId: 8996, + token: '0xtoken', + lockTx: '0xoldlock', + claimTx: '', + cancelTx: '', + cost: 5 + } + ] + }) + const { node, escrow } = buildFakes({ serviceJobInDb: job }) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + // the stale lock was refunded before charging again + expect(escrow.cancelExpiredLock.calledOnce).to.equal(true) + const out = await body(res) + expect(out[0].extendPayments).to.have.length(2) + expect(out[0].extendPayments[0].cancelTx).to.equal('0xcancel') + expect(out[0].extendPayments[1].claimTx).to.equal('0xclaim') + }) + + it('409 when an unresolved extension intent cannot be refunded (no double charge)', async () => { + const job = makeJob({ + extendPayments: [ + { + chainId: 8996, + token: '0xtoken', + lockTx: '0xoldlock', + claimTx: '', + cancelTx: '', + cost: 5 + } + ] + }) + const { node, escrow } = buildFakes({ serviceJobInDb: job }) + escrow.cancelExpiredLock.rejects(new Error('lock already claimed')) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(409) + // no new charge may be attempted while the old intent is unresolved + expect(escrow.createLock.called).to.equal(false) + expect(escrow.claimLock.called).to.equal(false) + }) }) describe('ServiceStartHandler', () => { diff --git a/src/test/unit/service/serviceJobsDatabase.test.ts b/src/test/unit/service/serviceJobsDatabase.test.ts index 5ab2e8321..ba71738d5 100644 --- a/src/test/unit/service/serviceJobsDatabase.test.ts +++ b/src/test/unit/service/serviceJobsDatabase.test.ts @@ -179,9 +179,14 @@ describe('Service Jobs Database', () => { it('refreshServiceLocks re-stamps a holder’s rows so they stay unstealable', async () => { const serviceId = `lock-svc-${Date.now()}-d` expect(await db.acquireServiceLock(serviceId, 'proc-A', STALE_MS)).to.equal(true) + // age the ORIGINAL stamp past the staleness window B will use below, so the + // refusal can only be explained by the refresh re-stamping the row — without the + // aging, a freshly-acquired row would be "unstealable" even if refresh did nothing + await new Promise((resolve) => setTimeout(resolve, 150)) await db.refreshServiceLocks('proc-A') - // B considers anything older than 5s stale — A's row was just re-stamped - expect(await db.acquireServiceLock(serviceId, 'proc-B', 5_000)).to.equal(false) + // B treats anything older than 100ms as stale: the original stamp (≥150ms old) + // would be stolen; the re-stamped one (a few ms old) must not be + expect(await db.acquireServiceLock(serviceId, 'proc-B', 100)).to.equal(false) await db.releaseServiceLock(serviceId, 'proc-A') }) }) diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts index 7db55c2f9..7917075b8 100644 --- a/src/test/unit/service/serviceRestartRace.test.ts +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -165,6 +165,11 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { ), 'Error status must be written to the DB' ).to.equal(true) + // the DB lease must be handed back on the failure path too + expect( + engine.db.releaseServiceLock.calledWith(SERVICE_ID, 'test-holder'), + 'DB lease must be released after the failed restart' + ).to.equal(true) }) it('restartService rejects a service whose payment was refunded (never claimed)', async () => { From d6153a0a735cd83991e8dc53cb958d9e6b20f71a Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Fri, 17 Jul 2026 07:43:06 +0300 Subject: [PATCH 09/10] add GetServicesCommand --- docs/API.md | 33 + docs/Ocean Node.postman_collection.json | 3630 +++++++++++------ docs/services.md | 1 + src/@types/commands.ts | 18 + .../core/handler/coreHandlersRegistry.ts | 2 + src/components/core/service/getServices.ts | 118 + src/components/core/service/index.ts | 1 + src/components/core/service/utils.ts | 24 + src/components/httpRoutes/compute.ts | 21 + src/test/integration/services.test.ts | 78 + src/test/unit/service/serviceHandlers.test.ts | 128 +- src/utils/constants.ts | 2 + 12 files changed, 2850 insertions(+), 1206 deletions(-) create mode 100644 src/components/core/service/getServices.ts diff --git a/docs/API.md b/docs/API.md index e00f0028b..f7c22db36 100644 --- a/docs/API.md +++ b/docs/API.md @@ -2015,6 +2015,39 @@ Array of `ServiceJob` (with `userData` stripped). --- +### `HTTP` GET /api/services/serviceList + +### `P2P` command: serviceList + +#### Description + +Node-wide service listing. **Authenticated but NOT owner-scoped** — any authenticated +consumer identity sees every owner's services. By default it returns only the services +**currently holding a resource reservation** (exactly what the engines count against the +shared pools): `Running`/`Restarting`/`Stopping`, the mid-start pipeline states, paid +`Error` (container died, restartable), and explicitly `Stopped` within the paid window. +`Expired` and never-paid jobs hold nothing and are not listed by default. + +#### Query Parameters + +| name | type | required | description | +| ----------------- | ------- | -------- | ----------- | +| consumerAddress | string | v | caller identity (any consumer) | +| nonce | string | v | request nonce | +| signature | string | v | signed message (or use an `Authorization` auth-token header) | +| status | number | | filter to ONE specific `ServiceStatusNumber` (any status, incl. `75` Expired); takes precedence over `includeAllStatuses` | +| includeAllStatuses | boolean | | `true` returns services in every status instead of only the resource-holding set | +| fromTimestamp | string | | only services created at/after this moment — ISO date (`2026-01-15T00:00:00Z`) or Unix timestamp (seconds or milliseconds) | + +#### Response (200) + +Array of `ServiceJob`, **listing-sanitized**: `userData`, `dockerCmd`, `dockerEntrypoint`, +`dockerfile` and `additionalDockerFiles` are stripped (identity, status, resources, +endpoints and payment metadata are kept). Use the owner-scoped `serviceStatus` to see a +service's own configuration. + +--- + ### `HTTP` POST /api/services/serviceExtend ### `P2P` command: serviceExtend diff --git a/docs/Ocean Node.postman_collection.json b/docs/Ocean Node.postman_collection.json index 46e7cf378..aab3d465c 100644 --- a/docs/Ocean Node.postman_collection.json +++ b/docs/Ocean Node.postman_collection.json @@ -1,1206 +1,2426 @@ { - "info": { - "name": "Ocean Node API", - "_postman_id": "ocean-node-http-api", - "description": "Complete collection of HTTP endpoints exposed by an Ocean Node's HTTP server.\n\nSet the `baseUrl` collection variable to your node (default `http://localhost:8000`). Many endpoints are authenticated with a signature over `consumerAddress` + `nonce` + `command`, or with an auth-token `Authorization` header. Convenience variables (`consumerAddress`, `signature`, `nonce`, `chainId`, `did`, `serviceId`, `jobId`, `token`, `bucketId`, `wallet`, `owner`) are provided as placeholders.\n\nGenerated from the route definitions in `src/components/httpRoutes/` and their handler implementations.", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" - }, - "variable": [ - { "key": "baseUrl", "value": "http://localhost:8000", "type": "string" }, - { "key": "consumerAddress", "value": "0x0000000000000000000000000000000000000000", "type": "string" }, - { "key": "signature", "value": "0x", "type": "string" }, - { "key": "nonce", "value": "1", "type": "string" }, - { "key": "chainId", "value": "8996", "type": "string" }, - { "key": "did", "value": "did:op:0000000000000000000000000000000000000000000000000000000000000000", "type": "string" }, - { "key": "serviceId", "value": "", "type": "string" }, - { "key": "jobId", "value": "", "type": "string" }, - { "key": "token", "value": "", "type": "string" }, - { "key": "bucketId", "value": "", "type": "string" }, - { "key": "wallet", "value": "0x0000000000000000000000000000000000000000", "type": "string" }, - { "key": "owner", "value": "0x0000000000000000000000000000000000000000", "type": "string" } - ], - "item": [ - { - "name": "Node Info", - "item": [ - { - "name": "Get Node Info", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/", - "host": ["{{baseUrl}}"], - "path": [""] - }, - "description": "Returns node identity and the list of available service endpoints (nodeId, chainIds, providerAddress, nodePublicKey, serviceEndpoints, software, version)." - } - } - ] - }, - { - "name": "Aquarius (DDO)", - "item": [ - { - "name": "Get DDO by DID", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/aquarius/assets/ddo/{{did}}", - "host": ["{{baseUrl}}"], - "path": ["api", "aquarius", "assets", "ddo", "{{did}}"] - }, - "description": "Retrieve the full DDO for a DID. Append an optional `/true` path segment to force a fresh lookup." - } - }, - { - "name": "Get DDO Metadata by DID", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/aquarius/assets/metadata/{{did}}", - "host": ["{{baseUrl}}"], - "path": ["api", "aquarius", "assets", "metadata", "{{did}}"] - }, - "description": "Retrieve DDO metadata for a DID. Optional trailing `/true` forces a fresh lookup." - } - }, - { - "name": "Query Assets", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"query\": {\n \"match_all\": {}\n },\n \"from\": 0,\n \"size\": 10\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/aquarius/assets/metadata/query", - "host": ["{{baseUrl}}"], - "path": ["api", "aquarius", "assets", "metadata", "query"] - }, - "description": "Query indexed assets. Body is a search query object (filter/query/sort/from/size)." - } - }, - { - "name": "Get DDO State", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/aquarius/state/ddo?did={{did}}", - "host": ["{{baseUrl}}"], - "path": ["api", "aquarius", "state", "ddo"], - "query": [ - { "key": "did", "value": "{{did}}" }, - { "key": "nft", "value": "", "disabled": true }, - { "key": "txId", "value": "", "disabled": true } - ] - }, - "description": "Query DDO state by `did`, `nft`, or `txId` (at least one required)." - } - }, - { - "name": "Validate DDO", - "request": { - "method": "POST", - "header": [ - { "key": "Content-Type", "value": "application/json" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { - "mode": "raw", - "raw": "{\n \"ddo\": {},\n \"publisherAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/aquarius/assets/ddo/validate", - "host": ["{{baseUrl}}"], - "path": ["api", "aquarius", "assets", "ddo", "validate"] - }, - "description": "Validate a DDO's schema and signature before publishing." - } - } - ] - }, - { - "name": "Provider", - "item": [ - { - "name": "Get Nonce", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/nonce?userAddress={{consumerAddress}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "nonce"], - "query": [{ "key": "userAddress", "value": "{{consumerAddress}}" }] - }, - "description": "Get the current nonce for a user address (used when signing requests)." - } - }, - { - "name": "Initialize (get fees)", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/initialize?documentId={{did}}&serviceId=&consumerAddress={{consumerAddress}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "initialize"], - "query": [ - { "key": "documentId", "value": "{{did}}" }, - { "key": "serviceId", "value": "" }, - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "validUntil", "value": "", "disabled": true }, - { "key": "policyServer", "value": "", "disabled": true } - ] - }, - "description": "Get the fees required to access an asset service." - } - }, - { - "name": "Encrypt", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/octet-stream" }], - "body": { "mode": "raw", "raw": "hello world" }, - "url": { - "raw": "{{baseUrl}}/api/services/encrypt?nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "encrypt"], - "query": [ - { "key": "nonce", "value": "{{nonce}}" }, - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" } - ] - }, - "description": "ECIES-encrypt the raw request body. Returns application/octet-stream. 25MB limit." - } - }, - { - "name": "Encrypt File", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"files\": {\n \"type\": \"url\",\n \"url\": \"https://example.com/file.txt\",\n \"method\": \"GET\"\n }\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/encryptFile?nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}&encryptMethod=ECIES", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "encryptFile"], - "query": [ - { "key": "nonce", "value": "{{nonce}}" }, - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "encryptMethod", "value": "ECIES" } - ] - }, - "description": "Encrypt a file (AES or ECIES). Accepts a StorageObject JSON, raw binary, or multipart. Returns encrypted bytes with X-Encrypted-By / X-Encrypted-Method headers." - } - }, - { - "name": "Decrypt", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"decrypterAddress\": \"{{consumerAddress}}\",\n \"chainId\": {{chainId}},\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"transactionId\": \"\",\n \"dataNftAddress\": \"\",\n \"encryptedDocument\": \"\",\n \"flags\": 0,\n \"documentHash\": \"\"\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/decrypt", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "decrypt"] - }, - "description": "Decrypt a DDO document. Returns the decrypted payload as text/plain." - } - }, - { - "name": "Download", - "request": { - "method": "GET", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/download?fileIndex=0&documentId={{did}}&serviceId=&transferTxId=&nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "download"], - "query": [ - { "key": "fileIndex", "value": "0" }, - { "key": "documentId", "value": "{{did}}" }, - { "key": "serviceId", "value": "" }, - { "key": "transferTxId", "value": "" }, - { "key": "nonce", "value": "{{nonce}}" }, - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "userdata", "value": "", "disabled": true }, - { "key": "policyServer", "value": "", "disabled": true } - ] - }, - "description": "Download an asset file after a valid transfer (order). Streams the file." - } - } - ] - }, - { - "name": "File Info", - "item": [ - { - "name": "File Info", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"did\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\"\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/fileInfo", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "fileInfo"] - }, - "description": "Get file info for an asset (by `did`+`serviceId`) or by a raw file `type` descriptor. `consumerAddress` is required for NODE_PERSISTENT_STORAGE ACL gating." - } - } - ] - }, - { - "name": "Compute", - "item": [ - { - "name": "Get Compute Environments", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/computeEnvironments?chainId={{chainId}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "computeEnvironments"], - "query": [ - { "key": "chainId", "value": "{{chainId}}" }, - { "key": "node", "value": "", "disabled": true } - ] - }, - "description": "List available compute environments (optionally filtered by chain)." - } - }, - { - "name": "Initialize Compute", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [\n { \"documentId\": \"{{did}}\", \"serviceId\": \"\" }\n ]\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/initializeCompute", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "initializeCompute"] - }, - "description": "Validate algorithm + datasets and return a price/initialization quote." - } - }, - { - "name": "Start Compute (paid)", - "request": { - "method": "POST", - "header": [ - { "key": "Content-Type", "value": "application/json" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { - "mode": "raw", - "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"environment\": \"\",\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [ { \"documentId\": \"{{did}}\", \"serviceId\": \"\" } ],\n \"maxJobDuration\": 3600,\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/compute", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "compute"] - }, - "description": "Start a paid compute job. Optional fields: policyServer, metadata, additionalViewers, queueMaxWaitTime, encryptedDockerRegistryAuth, output, outputBucketId." - } - }, - { - "name": "Start Free Compute", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"environment\": \"\",\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [],\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"maxJobDuration\": 600\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/freeCompute", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "freeCompute"] - }, - "description": "Start a free compute job (no payment) on a free-tier environment." - } - }, - { - "name": "Get Compute Status", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/compute?consumerAddress={{consumerAddress}}&jobId={{jobId}}&agreementId=", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "compute"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "jobId", "value": "{{jobId}}" }, - { "key": "agreementId", "value": "" }, - { "key": "node", "value": "", "disabled": true } - ] - }, - "description": "Get the status of a compute job." - } - }, - { - "name": "Stop Compute", - "request": { - "method": "PUT", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/compute?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}&jobId={{jobId}}&agreementId=", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "compute"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" }, - { "key": "jobId", "value": "{{jobId}}" }, - { "key": "agreementId", "value": "" }, - { "key": "node", "value": "", "disabled": true } - ] - }, - "description": "Stop a running compute job (parameters are query strings)." - } - }, - { - "name": "Get Compute Result", - "request": { - "method": "GET", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/computeResult?consumerAddress={{consumerAddress}}&jobId={{jobId}}&index=0&signature={{signature}}&nonce={{nonce}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "computeResult"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "jobId", "value": "{{jobId}}" }, - { "key": "index", "value": "0" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" }, - { "key": "node", "value": "", "disabled": true } - ] - }, - "description": "Download a compute job result by index. Streams the result file." - } - }, - { - "name": "Get Compute Streamable Logs", - "request": { - "method": "GET", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/computeStreamableLogs?consumerAddress={{consumerAddress}}&jobId={{jobId}}&signature={{signature}}&nonce={{nonce}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "computeStreamableLogs"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "jobId", "value": "{{jobId}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" }, - { "key": "node", "value": "", "disabled": true } - ] - }, - "description": "Stream real-time logs from a running compute job (404 if not running)." - } - } - ] - }, - { - "name": "Service on Demand", - "item": [ - { - "name": "Get Service Templates", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/serviceTemplates?chainId={{chainId}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "serviceTemplates"], - "query": [ - { "key": "chainId", "value": "{{chainId}}" }, - { "key": "node", "value": "", "disabled": true } - ] - }, - "description": "List operator-published service templates (sanitized). Not authenticated." - } - }, - { - "name": "Start Service", - "request": { - "method": "POST", - "header": [ - { "key": "Content-Type", "value": "application/json" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { - "mode": "raw", - "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"environment\": \"\",\n \"image\": \"nginxinc/nginx-unprivileged\",\n \"tag\": \"alpine\",\n \"exposedPorts\": [8080],\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"duration\": 3600,\n \"userData\": \"\",\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/serviceStart", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "serviceStart"] - }, - "description": "Launch a long-running service container, paid via escrow. Optional: checksum, dockerfile, additionalDockerFiles, dockerCmd, dockerEntrypoint. Services must listen on a high port (>1024)." - } - }, - { - "name": "Get Service Status", - "request": { - "method": "GET", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/serviceStatus?consumerAddress={{consumerAddress}}&nonce={{nonce}}&signature={{signature}}&serviceId={{serviceId}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "serviceStatus"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "nonce", "value": "{{nonce}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "serviceId", "value": "{{serviceId}}" }, - { "key": "node", "value": "", "disabled": true } - ] - }, - "description": "Read service status/endpoints. Authenticated and owner-scoped. Omit serviceId to list all owned services." - } - }, - { - "name": "Extend Service", - "request": { - "method": "POST", - "header": [ - { "key": "Content-Type", "value": "application/json" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { - "mode": "raw", - "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\",\n \"additionalDuration\": 1800,\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/serviceExtend", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "serviceExtend"] - }, - "description": "Pay to push the service expiry further out (additionalDuration in seconds, must be positive)." - } - }, - { - "name": "Restart Service", - "request": { - "method": "POST", - "header": [ - { "key": "Content-Type", "value": "application/json" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { - "mode": "raw", - "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\",\n \"userData\": \"\"\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/serviceRestart", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "serviceRestart"] - }, - "description": "Recreate the service container (no extra charge), keeping the same hostPort and expiresAt. Optional userData replaces stored env vars." - } - }, - { - "name": "Stop Service", - "request": { - "method": "POST", - "header": [ - { "key": "Content-Type", "value": "application/json" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { - "mode": "raw", - "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\"\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/serviceStop", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "serviceStop"] - }, - "description": "Tear down the service container and network and release resources. Owner-gated." - } - }, - { - "name": "Get Service Streamable Logs", - "request": { - "method": "GET", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/serviceStreamableLogs?consumerAddress={{consumerAddress}}&serviceId={{serviceId}}&signature={{signature}}&nonce={{nonce}}&since=1h", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "serviceStreamableLogs"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "serviceId", "value": "{{serviceId}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" }, - { - "key": "since", - "value": "1h", - "description": "Optional: Unix timestamp (seconds) or relative duration like 30s/45m/2h/7d. Omit for full history + live follow." - }, - { "key": "node", "value": "", "disabled": true } - ] - }, - "description": "Stream real-time stdout/stderr logs from a service container. Authenticated and owner-scoped (401 if consumerAddress is not the owner). Available while Running or Error; 404 otherwise. Optional `since` skips history and starts from a recent point (Unix timestamp or relative duration e.g. 1h)." - } - } - ] - }, - { - "name": "Persistent Storage", - "item": [ - { - "name": "Create Bucket", - "request": { - "method": "POST", - "header": [ - { "key": "Content-Type", "value": "application/json" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { - "mode": "raw", - "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"label\": \"my-bucket\",\n \"accessLists\": []\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/persistentStorage/buckets", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "persistentStorage", "buckets"] - }, - "description": "Create a new persistent storage bucket owned by consumerAddress." - } - }, - { - "name": "Update Bucket", - "request": { - "method": "PATCH", - "header": [ - { "key": "Content-Type", "value": "application/json" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { "mode": "raw", "raw": "{\n \"label\": \"renamed-bucket\"\n}" }, - "url": { - "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "persistentStorage", "buckets", "{{bucketId}}"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" } - ] - }, - "description": "Update a bucket (e.g. set its label)." - } - }, - { - "name": "List Buckets", - "request": { - "method": "GET", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/persistentStorage/buckets?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}&owner={{owner}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "persistentStorage", "buckets"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" }, - { "key": "owner", "value": "{{owner}}" } - ] - }, - "description": "List buckets for an owner (filtered by access lists for the calling consumer)." - } - }, - { - "name": "List Files in Bucket", - "request": { - "method": "GET", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "persistentStorage", "buckets", "{{bucketId}}", "files"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" } - ] - }, - "description": "List all files in a bucket." - } - }, - { - "name": "Get File Object", - "request": { - "method": "GET", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt/object?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "persistentStorage", "buckets", "{{bucketId}}", "files", "myfile.txt", "object"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" } - ] - }, - "description": "Get the file object metadata for a file in a bucket." - } - }, - { - "name": "Upload File", - "request": { - "method": "POST", - "header": [ - { "key": "Content-Type", "value": "application/octet-stream" }, - { "key": "Authorization", "value": "{{token}}", "disabled": true } - ], - "body": { "mode": "raw", "raw": "" }, - "url": { - "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "persistentStorage", "buckets", "{{bucketId}}", "files", "myfile.txt"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" } - ] - }, - "description": "Upload a file to a bucket. Body is the raw file bytes (supports chunked uploads)." - } - }, - { - "name": "Delete File", - "request": { - "method": "DELETE", - "header": [{ "key": "Authorization", "value": "{{token}}", "disabled": true }], - "url": { - "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "persistentStorage", "buckets", "{{bucketId}}", "files", "myfile.txt"], - "query": [ - { "key": "consumerAddress", "value": "{{consumerAddress}}" }, - { "key": "signature", "value": "{{signature}}" }, - { "key": "nonce", "value": "{{nonce}}" } - ] - }, - "description": "Delete a file from a bucket. Returns { success: true }." - } - } - ] - }, - { - "name": "Escrow", - "item": [ - { - "name": "Get Escrow Events", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/escrow/events?chainId={{chainId}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "escrow", "events"], - "query": [ - { "key": "chainId", "value": "{{chainId}}" }, - { "key": "eventType", "value": "", "disabled": true }, - { "key": "payer", "value": "", "disabled": true }, - { "key": "payee", "value": "", "disabled": true }, - { "key": "token", "value": "", "disabled": true }, - { "key": "jobId", "value": "", "disabled": true }, - { "key": "txId", "value": "", "disabled": true }, - { "key": "offset", "value": "", "disabled": true }, - { "key": "size", "value": "", "disabled": true } - ] - }, - "description": "Query escrow contract events with optional filters (eventType e.g. Lock/Claimed/Canceled/Deposit/Withdraw/Auth)." - } - } - ] - }, - { - "name": "Access Lists", - "item": [ - { - "name": "Search Access Lists by Wallet", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/accesslists?wallet={{wallet}}&chainId={{chainId}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "accesslists"], - "query": [ - { "key": "wallet", "value": "{{wallet}}" }, - { "key": "chainId", "value": "{{chainId}}", "disabled": true } - ] - }, - "description": "Search access lists that include a given wallet address." - } - }, - { - "name": "Get Access List", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/accesslists/{{chainId}}/0xContractAddress", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "accesslists", "{{chainId}}", "0xContractAddress"] - }, - "description": "Get a specific access list by chainId and contract address." - } - } - ] - }, - { - "name": "Auth", - "item": [ - { - "name": "Create Auth Token", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"validUntil\": null,\n \"chainId\": {{chainId}}\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/auth/token", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "auth", "token"] - }, - "description": "Create an auth token (returned token can be used as the Authorization header on authenticated routes)." - } - }, - { - "name": "Invalidate Auth Token", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"token\": \"{{token}}\",\n \"nonce\": \"{{nonce}}\",\n \"chainId\": {{chainId}}\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/auth/token/invalidate", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "auth", "token", "invalidate"] - }, - "description": "Invalidate an existing auth token." - } - } - ] - }, - { - "name": "Policy Server", - "item": [ - { - "name": "Policy Server Passthrough", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"policyServerPassthrough\": {}\n}" }, - "url": { - "raw": "{{baseUrl}}/api/services/PolicyServerPassthrough", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "PolicyServerPassthrough"] - }, - "description": "Pass arbitrary data through to the configured policy server. Streams the response." - } - }, - { - "name": "Initialize PS Verification", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"documentId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"policyServer\": {}\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/services/initializePSVerification", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "initializePSVerification"] - }, - "description": "Initialize a policy server verification flow. Streams the response." - } - } - ] - }, - { - "name": "Admin", - "item": [ - { - "name": "Fetch Config", - "request": { - "method": "GET", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/admin/config", - "host": ["{{baseUrl}}"], - "path": ["api", "admin", "config"] - }, - "description": "Fetch the node configuration (admin-only). Note: this GET reads auth fields from the JSON body." - } - }, - { - "name": "Update Config", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"config\": {}\n}" - }, - "url": { - "raw": "{{baseUrl}}/api/admin/config/update", - "host": ["{{baseUrl}}"], - "path": ["api", "admin", "config", "update"] - }, - "description": "Update the node configuration (admin-only)." - } - } - ] - }, - { - "name": "Queue & Jobs", - "item": [ - { - "name": "Get Index Queue", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/indexQueue", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "indexQueue"] - }, - "description": "Get the current indexing queue." - } - }, - { - "name": "Get Jobs for Identifier", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/api/services/jobs/{{jobId}}", - "host": ["{{baseUrl}}"], - "path": ["api", "services", "jobs", "{{jobId}}"] - }, - "description": "Get the indexer job pool for a given job identifier." - } - } - ] - }, - { - "name": "P2P Peers", - "item": [ - { - "name": "Get P2P Network Stats", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/getP2pNetworkStats", - "host": ["{{baseUrl}}"], - "path": ["getP2pNetworkStats"] - }, - "description": "Get P2P network statistics." - } - }, - { - "name": "Get P2P Peers", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/getP2PPeers", - "host": ["{{baseUrl}}"], - "path": ["getP2PPeers"] - }, - "description": "List all connected P2P peers." - } - }, - { - "name": "Get P2P Peer", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/getP2PPeer?peerId=", - "host": ["{{baseUrl}}"], - "path": ["getP2PPeer"], - "query": [{ "key": "peerId", "value": "" }] - }, - "description": "Get details for a specific peer by peerId." - } - }, - { - "name": "Find Peer", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/findPeer?peerId=&timeout=10000", - "host": ["{{baseUrl}}"], - "path": ["findPeer"], - "query": [ - { "key": "peerId", "value": "" }, - { "key": "timeout", "value": "10000", "disabled": true } - ] - }, - "description": "Find a peer's multiaddress by peerId." - } - } - ] - }, - { - "name": "DIDs / Providers", - "item": [ - { - "name": "Get Providers for String", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{baseUrl}}/getProvidersForString?input={{did}}", - "host": ["{{baseUrl}}"], - "path": ["getProvidersForString"], - "query": [{ "key": "input", "value": "{{did}}" }] - }, - "description": "Get the providers (peers) that serve a single string/CID." - } - }, - { - "name": "Get Providers for Strings", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "[\n \"did:op:...\",\n \"did:op:...\"\n]" }, - "url": { - "raw": "{{baseUrl}}/getProvidersForStrings?timeout=10000", - "host": ["{{baseUrl}}"], - "path": ["getProvidersForStrings"], - "query": [{ "key": "timeout", "value": "10000", "disabled": true }] - }, - "description": "Batch lookup: get providers for an array of strings/CIDs. Body is a JSON array of strings." - } - } - ] - }, - { - "name": "Logs", - "item": [ - { - "name": "Get Logs", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" - }, - "url": { - "raw": "{{baseUrl}}/logs?maxLogs=100&page=1", - "host": ["{{baseUrl}}"], - "path": ["logs"], - "query": [ - { "key": "maxLogs", "value": "100" }, - { "key": "page", "value": "1" }, - { "key": "moduleName", "value": "", "disabled": true }, - { "key": "level", "value": "", "disabled": true }, - { "key": "startTime", "value": "", "disabled": true }, - { "key": "endTime", "value": "", "disabled": true } - ] - }, - "description": "Retrieve node logs (admin auth via signed body). Filter/paginate with query params." - } - }, - { - "name": "Get Log by ID", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { - "mode": "raw", - "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"logId\": \"123\"\n}" - }, - "url": { - "raw": "{{baseUrl}}/log/123", - "host": ["{{baseUrl}}"], - "path": ["log", "123"] - }, - "description": "Retrieve a single log entry by id (logId in body must match the :id path param)." - } - } - ] - }, - { - "name": "Direct Command (P2P)", - "description": "POST /directCommand is a single endpoint that dispatches to a protocol command handler chosen by the `command` field. It can run locally or be forwarded to another peer via `node`/`multiAddrs`. Below are representative command bodies; any PROTOCOL_COMMANDS value is accepted (download, encrypt, encryptFile, decryptDDO, getDDO, query, nonce, status, detailedStatus, findDDO, getFees, fileInfo, validateDDO, getComputeEnvironments, startCompute, freeStartCompute, stopCompute, getComputeStatus, getComputeStreamableLogs, getComputeResult, initializeCompute, reindexTx, reindexChain, collectFees, PolicyServerPassthrough, getP2PPeer(s), getP2PNetworkStats, findPeer, createAuthToken, invalidateAuthToken, fetchConfig, pushConfig, getLogs, jobs, persistentStorage*, getAccessList, searchAccessList, getEscrowEvents, serviceGetTemplates, serviceStart, serviceStop, serviceRestart, serviceGetStatus, serviceExtend, serviceGetStreamableLogs, stopNode, ...).", - "item": [ - { - "name": "getDDO", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"getDDO\",\n \"id\": \"{{did}}\"\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Fetch a DDO by id over the command interface." - } - }, - { - "name": "nonce", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"nonce\",\n \"address\": \"{{consumerAddress}}\"\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Get the nonce for an address." - } - }, - { - "name": "status", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"status\"\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Get node status. Use \"command\": \"detailedStatus\" for the extended report." - } - }, - { - "name": "query", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"query\",\n \"query\": { \"query\": { \"match_all\": {} }, \"from\": 0, \"size\": 10 }\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Query indexed assets." - } - }, - { - "name": "getFees", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"getFees\",\n \"ddoId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"validUntil\": 0\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Get the provider fees for an asset service." - } - }, - { - "name": "fileInfo", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"fileInfo\",\n \"did\": \"{{did}}\",\n \"serviceId\": \"\"\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Get file info for an asset or a raw file descriptor (type/url/...)." - } - }, - { - "name": "download", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"download\",\n \"fileIndex\": 0,\n \"documentId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"transferTxId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Download an asset file (streams the response)." - } - }, - { - "name": "encrypt", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"encrypt\",\n \"blob\": \"hello\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"encoding\": \"string\",\n \"encryptionType\": \"ECIES\"\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Encrypt a blob (AES or ECIES)." - } - }, - { - "name": "decryptDDO", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"decryptDDO\",\n \"decrypterAddress\": \"{{consumerAddress}}\",\n \"chainId\": {{chainId}},\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"transactionId\": \"\",\n \"dataNftAddress\": \"\"\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Decrypt a DDO document." - } - }, - { - "name": "findDDO", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"findDDO\",\n \"id\": \"{{did}}\"\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Find which peers can serve a DDO." - } - }, - { - "name": "validateDDO", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"validateDDO\",\n \"ddo\": {}\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Validate a DDO." - } - }, - { - "name": "getComputeEnvironments", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"getComputeEnvironments\",\n \"chainId\": {{chainId}}\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "List compute environments over the command interface." - } - }, - { - "name": "getEscrowEvents", - "request": { - "method": "POST", - "header": [{ "key": "Content-Type", "value": "application/json" }], - "body": { "mode": "raw", "raw": "{\n \"command\": \"getEscrowEvents\",\n \"chainId\": {{chainId}}\n}" }, - "url": { "raw": "{{baseUrl}}/directCommand", "host": ["{{baseUrl}}"], "path": ["directCommand"] }, - "description": "Query escrow events over the command interface." - } - } - ] - } - ] -} + "info": { + "name": "Ocean Node API", + "_postman_id": "ocean-node-http-api", + "description": "Complete collection of HTTP endpoints exposed by an Ocean Node's HTTP server.\n\nSet the `baseUrl` collection variable to your node (default `http://localhost:8000`). Many endpoints are authenticated with a signature over `consumerAddress` + `nonce` + `command`, or with an auth-token `Authorization` header. Convenience variables (`consumerAddress`, `signature`, `nonce`, `chainId`, `did`, `serviceId`, `jobId`, `token`, `bucketId`, `wallet`, `owner`) are provided as placeholders.\n\nGenerated from the route definitions in `src/components/httpRoutes/` and their handler implementations.", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "variable": [ + { + "key": "baseUrl", + "value": "http://localhost:8000", + "type": "string" + }, + { + "key": "consumerAddress", + "value": "0x0000000000000000000000000000000000000000", + "type": "string" + }, + { + "key": "signature", + "value": "0x", + "type": "string" + }, + { + "key": "nonce", + "value": "1", + "type": "string" + }, + { + "key": "chainId", + "value": "8996", + "type": "string" + }, + { + "key": "did", + "value": "did:op:0000000000000000000000000000000000000000000000000000000000000000", + "type": "string" + }, + { + "key": "serviceId", + "value": "", + "type": "string" + }, + { + "key": "jobId", + "value": "", + "type": "string" + }, + { + "key": "token", + "value": "", + "type": "string" + }, + { + "key": "bucketId", + "value": "", + "type": "string" + }, + { + "key": "wallet", + "value": "0x0000000000000000000000000000000000000000", + "type": "string" + }, + { + "key": "owner", + "value": "0x0000000000000000000000000000000000000000", + "type": "string" + } + ], + "item": [ + { + "name": "Node Info", + "item": [ + { + "name": "Get Node Info", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "" + ] + }, + "description": "Returns node identity and the list of available service endpoints (nodeId, chainIds, providerAddress, nodePublicKey, serviceEndpoints, software, version)." + } + } + ] + }, + { + "name": "Aquarius (DDO)", + "item": [ + { + "name": "Get DDO by DID", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/aquarius/assets/ddo/{{did}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "assets", + "ddo", + "{{did}}" + ] + }, + "description": "Retrieve the full DDO for a DID. Append an optional `/true` path segment to force a fresh lookup." + } + }, + { + "name": "Get DDO Metadata by DID", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/aquarius/assets/metadata/{{did}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "assets", + "metadata", + "{{did}}" + ] + }, + "description": "Retrieve DDO metadata for a DID. Optional trailing `/true` forces a fresh lookup." + } + }, + { + "name": "Query Assets", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"query\": {\n \"match_all\": {}\n },\n \"from\": 0,\n \"size\": 10\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/aquarius/assets/metadata/query", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "assets", + "metadata", + "query" + ] + }, + "description": "Query indexed assets. Body is a search query object (filter/query/sort/from/size)." + } + }, + { + "name": "Get DDO State", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/aquarius/state/ddo?did={{did}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "state", + "ddo" + ], + "query": [ + { + "key": "did", + "value": "{{did}}" + }, + { + "key": "nft", + "value": "", + "disabled": true + }, + { + "key": "txId", + "value": "", + "disabled": true + } + ] + }, + "description": "Query DDO state by `did`, `nft`, or `txId` (at least one required)." + } + }, + { + "name": "Validate DDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"ddo\": {},\n \"publisherAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/aquarius/assets/ddo/validate", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "aquarius", + "assets", + "ddo", + "validate" + ] + }, + "description": "Validate a DDO's schema and signature before publishing." + } + } + ] + }, + { + "name": "Provider", + "item": [ + { + "name": "Get Nonce", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/nonce?userAddress={{consumerAddress}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "nonce" + ], + "query": [ + { + "key": "userAddress", + "value": "{{consumerAddress}}" + } + ] + }, + "description": "Get the current nonce for a user address (used when signing requests)." + } + }, + { + "name": "Initialize (get fees)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/initialize?documentId={{did}}&serviceId=&consumerAddress={{consumerAddress}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "initialize" + ], + "query": [ + { + "key": "documentId", + "value": "{{did}}" + }, + { + "key": "serviceId", + "value": "" + }, + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "validUntil", + "value": "", + "disabled": true + }, + { + "key": "policyServer", + "value": "", + "disabled": true + } + ] + }, + "description": "Get the fees required to access an asset service." + } + }, + { + "name": "Encrypt", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/octet-stream" + } + ], + "body": { + "mode": "raw", + "raw": "hello world" + }, + "url": { + "raw": "{{baseUrl}}/api/services/encrypt?nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "encrypt" + ], + "query": [ + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + } + ] + }, + "description": "ECIES-encrypt the raw request body. Returns application/octet-stream. 25MB limit." + } + }, + { + "name": "Encrypt File", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"files\": {\n \"type\": \"url\",\n \"url\": \"https://example.com/file.txt\",\n \"method\": \"GET\"\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/encryptFile?nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}&encryptMethod=ECIES", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "encryptFile" + ], + "query": [ + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "encryptMethod", + "value": "ECIES" + } + ] + }, + "description": "Encrypt a file (AES or ECIES). Accepts a StorageObject JSON, raw binary, or multipart. Returns encrypted bytes with X-Encrypted-By / X-Encrypted-Method headers." + } + }, + { + "name": "Decrypt", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"decrypterAddress\": \"{{consumerAddress}}\",\n \"chainId\": {{chainId}},\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"transactionId\": \"\",\n \"dataNftAddress\": \"\",\n \"encryptedDocument\": \"\",\n \"flags\": 0,\n \"documentHash\": \"\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/decrypt", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "decrypt" + ] + }, + "description": "Decrypt a DDO document. Returns the decrypted payload as text/plain." + } + }, + { + "name": "Download", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/download?fileIndex=0&documentId={{did}}&serviceId=&transferTxId=&nonce={{nonce}}&consumerAddress={{consumerAddress}}&signature={{signature}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "download" + ], + "query": [ + { + "key": "fileIndex", + "value": "0" + }, + { + "key": "documentId", + "value": "{{did}}" + }, + { + "key": "serviceId", + "value": "" + }, + { + "key": "transferTxId", + "value": "" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "userdata", + "value": "", + "disabled": true + }, + { + "key": "policyServer", + "value": "", + "disabled": true + } + ] + }, + "description": "Download an asset file after a valid transfer (order). Streams the file." + } + } + ] + }, + { + "name": "File Info", + "item": [ + { + "name": "File Info", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"did\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/fileInfo", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "fileInfo" + ] + }, + "description": "Get file info for an asset (by `did`+`serviceId`) or by a raw file `type` descriptor. `consumerAddress` is required for NODE_PERSISTENT_STORAGE ACL gating." + } + } + ] + }, + { + "name": "Compute", + "item": [ + { + "name": "Get Compute Environments", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/computeEnvironments?chainId={{chainId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "computeEnvironments" + ], + "query": [ + { + "key": "chainId", + "value": "{{chainId}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "List available compute environments (optionally filtered by chain)." + } + }, + { + "name": "Initialize Compute", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [\n { \"documentId\": \"{{did}}\", \"serviceId\": \"\" }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/initializeCompute", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "initializeCompute" + ] + }, + "description": "Validate algorithm + datasets and return a price/initialization quote." + } + }, + { + "name": "Start Compute (paid)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"environment\": \"\",\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [ { \"documentId\": \"{{did}}\", \"serviceId\": \"\" } ],\n \"maxJobDuration\": 3600,\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/compute", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "compute" + ] + }, + "description": "Start a paid compute job. Optional fields: policyServer, metadata, additionalViewers, queueMaxWaitTime, encryptedDockerRegistryAuth, output, outputBucketId." + } + }, + { + "name": "Start Free Compute", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"environment\": \"\",\n \"algorithm\": { \"documentId\": \"{{did}}\", \"serviceId\": \"\" },\n \"datasets\": [],\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"maxJobDuration\": 600\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/freeCompute", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "freeCompute" + ] + }, + "description": "Start a free compute job (no payment) on a free-tier environment." + } + }, + { + "name": "Get Compute Status", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/compute?consumerAddress={{consumerAddress}}&jobId={{jobId}}&agreementId=", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "compute" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "jobId", + "value": "{{jobId}}" + }, + { + "key": "agreementId", + "value": "" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Get the status of a compute job." + } + }, + { + "name": "Stop Compute", + "request": { + "method": "PUT", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/compute?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}&jobId={{jobId}}&agreementId=", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "compute" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "jobId", + "value": "{{jobId}}" + }, + { + "key": "agreementId", + "value": "" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Stop a running compute job (parameters are query strings)." + } + }, + { + "name": "Get Compute Result", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/computeResult?consumerAddress={{consumerAddress}}&jobId={{jobId}}&index=0&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "computeResult" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "jobId", + "value": "{{jobId}}" + }, + { + "key": "index", + "value": "0" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Download a compute job result by index. Streams the result file." + } + }, + { + "name": "Get Compute Streamable Logs", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/computeStreamableLogs?consumerAddress={{consumerAddress}}&jobId={{jobId}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "computeStreamableLogs" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "jobId", + "value": "{{jobId}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Stream real-time logs from a running compute job (404 if not running)." + } + } + ] + }, + { + "name": "Service on Demand", + "item": [ + { + "name": "Get Service Templates", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/serviceTemplates?chainId={{chainId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceTemplates" + ], + "query": [ + { + "key": "chainId", + "value": "{{chainId}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "List operator-published service templates (sanitized). Not authenticated." + } + }, + { + "name": "Start Service", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"environment\": \"\",\n \"image\": \"nginxinc/nginx-unprivileged\",\n \"tag\": \"alpine\",\n \"exposedPorts\": [8080],\n \"resources\": [ { \"id\": \"cpu\", \"amount\": 1 }, { \"id\": \"ram\", \"amount\": 1 } ],\n \"duration\": 3600,\n \"userData\": \"\",\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/serviceStart", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceStart" + ] + }, + "description": "Launch a long-running service container, paid via escrow. Optional: checksum, dockerfile, additionalDockerFiles, dockerCmd, dockerEntrypoint. Services must listen on a high port (>1024)." + } + }, + { + "name": "Get Service Status", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/serviceStatus?consumerAddress={{consumerAddress}}&nonce={{nonce}}&signature={{signature}}&serviceId={{serviceId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceStatus" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "serviceId", + "value": "{{serviceId}}" + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Read service status/endpoints. Authenticated and owner-scoped. Omit serviceId to list all owned services." + } + }, + { + "name": "List Services", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/serviceList?consumerAddress={{consumerAddress}}&nonce={{nonce}}&signature={{signature}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceList" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "status", + "value": "70", + "description": "one specific ServiceStatusNumber (e.g. 70 = Stopped, 75 = Expired); takes precedence over includeAllStatuses", + "disabled": true + }, + { + "key": "includeAllStatuses", + "value": "true", + "description": "every status instead of only the resource-holding set", + "disabled": true + }, + { + "key": "fromTimestamp", + "value": "2026-01-01T00:00:00Z", + "description": "only services created at/after this moment (ISO date or Unix timestamp)", + "disabled": true + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Node-wide service listing (authenticated, NOT owner-scoped). Default: only services currently holding a resource reservation (Running/Restarting/Stopping, mid-start states, paid Error, Stopped within the paid window). Filter with status / includeAllStatuses / fromTimestamp. Output is listing-sanitized: no userData, dockerCmd, dockerEntrypoint or Dockerfile." + } + }, + { + "name": "Extend Service", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\",\n \"additionalDuration\": 1800,\n \"payment\": { \"chainId\": {{chainId}}, \"token\": \"0x...\" }\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/serviceExtend", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceExtend" + ] + }, + "description": "Pay to push the service expiry further out (additionalDuration in seconds, must be positive)." + } + }, + { + "name": "Restart Service", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\",\n \"userData\": \"\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/serviceRestart", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceRestart" + ] + }, + "description": "Recreate the service container (no extra charge), keeping the same hostPort and expiresAt. Optional userData replaces stored env vars." + } + }, + { + "name": "Stop Service", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"serviceId\": \"{{serviceId}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/serviceStop", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceStop" + ] + }, + "description": "Tear down the service container and network and release resources. Owner-gated." + } + }, + { + "name": "Get Service Streamable Logs", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/serviceStreamableLogs?consumerAddress={{consumerAddress}}&serviceId={{serviceId}}&signature={{signature}}&nonce={{nonce}}&since=1h", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "serviceStreamableLogs" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "serviceId", + "value": "{{serviceId}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "since", + "value": "1h", + "description": "Optional: Unix timestamp (seconds) or relative duration like 30s/45m/2h/7d. Omit for full history + live follow." + }, + { + "key": "node", + "value": "", + "disabled": true + } + ] + }, + "description": "Stream real-time stdout/stderr logs from a service container. Authenticated and owner-scoped (401 if consumerAddress is not the owner). Available while Running or Error; 404 otherwise. Optional `since` skips history and starts from a recent point (Unix timestamp or relative duration e.g. 1h)." + } + } + ] + }, + { + "name": "Persistent Storage", + "item": [ + { + "name": "Create Bucket", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"label\": \"my-bucket\",\n \"accessLists\": []\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets" + ] + }, + "description": "Create a new persistent storage bucket owned by consumerAddress." + } + }, + { + "name": "Update Bucket", + "request": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"label\": \"renamed-bucket\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "Update a bucket (e.g. set its label)." + } + }, + { + "name": "List Buckets", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}&owner={{owner}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + }, + { + "key": "owner", + "value": "{{owner}}" + } + ] + }, + "description": "List buckets for an owner (filtered by access lists for the calling consumer)." + } + }, + { + "name": "List Files in Bucket", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}", + "files" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "List all files in a bucket." + } + }, + { + "name": "Get File Object", + "request": { + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt/object?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}", + "files", + "myfile.txt", + "object" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "Get the file object metadata for a file in a bucket." + } + }, + { + "name": "Upload File", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/octet-stream" + }, + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "" + }, + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}", + "files", + "myfile.txt" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "Upload a file to a bucket. Body is the raw file bytes (supports chunked uploads)." + } + }, + { + "name": "Delete File", + "request": { + "method": "DELETE", + "header": [ + { + "key": "Authorization", + "value": "{{token}}", + "disabled": true + } + ], + "url": { + "raw": "{{baseUrl}}/api/services/persistentStorage/buckets/{{bucketId}}/files/myfile.txt?consumerAddress={{consumerAddress}}&signature={{signature}}&nonce={{nonce}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "persistentStorage", + "buckets", + "{{bucketId}}", + "files", + "myfile.txt" + ], + "query": [ + { + "key": "consumerAddress", + "value": "{{consumerAddress}}" + }, + { + "key": "signature", + "value": "{{signature}}" + }, + { + "key": "nonce", + "value": "{{nonce}}" + } + ] + }, + "description": "Delete a file from a bucket. Returns { success: true }." + } + } + ] + }, + { + "name": "Escrow", + "item": [ + { + "name": "Get Escrow Events", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/escrow/events?chainId={{chainId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "escrow", + "events" + ], + "query": [ + { + "key": "chainId", + "value": "{{chainId}}" + }, + { + "key": "eventType", + "value": "", + "disabled": true + }, + { + "key": "payer", + "value": "", + "disabled": true + }, + { + "key": "payee", + "value": "", + "disabled": true + }, + { + "key": "token", + "value": "", + "disabled": true + }, + { + "key": "jobId", + "value": "", + "disabled": true + }, + { + "key": "txId", + "value": "", + "disabled": true + }, + { + "key": "offset", + "value": "", + "disabled": true + }, + { + "key": "size", + "value": "", + "disabled": true + } + ] + }, + "description": "Query escrow contract events with optional filters (eventType e.g. Lock/Claimed/Canceled/Deposit/Withdraw/Auth)." + } + } + ] + }, + { + "name": "Access Lists", + "item": [ + { + "name": "Search Access Lists by Wallet", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/accesslists?wallet={{wallet}}&chainId={{chainId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "accesslists" + ], + "query": [ + { + "key": "wallet", + "value": "{{wallet}}" + }, + { + "key": "chainId", + "value": "{{chainId}}", + "disabled": true + } + ] + }, + "description": "Search access lists that include a given wallet address." + } + }, + { + "name": "Get Access List", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/accesslists/{{chainId}}/0xContractAddress", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "accesslists", + "{{chainId}}", + "0xContractAddress" + ] + }, + "description": "Get a specific access list by chainId and contract address." + } + } + ] + }, + { + "name": "Auth", + "item": [ + { + "name": "Create Auth Token", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"validUntil\": null,\n \"chainId\": {{chainId}}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/auth/token", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "auth", + "token" + ] + }, + "description": "Create an auth token (returned token can be used as the Authorization header on authenticated routes)." + } + }, + { + "name": "Invalidate Auth Token", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"token\": \"{{token}}\",\n \"nonce\": \"{{nonce}}\",\n \"chainId\": {{chainId}}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/auth/token/invalidate", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "auth", + "token", + "invalidate" + ] + }, + "description": "Invalidate an existing auth token." + } + } + ] + }, + { + "name": "Policy Server", + "item": [ + { + "name": "Policy Server Passthrough", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"policyServerPassthrough\": {}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/PolicyServerPassthrough", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "PolicyServerPassthrough" + ] + }, + "description": "Pass arbitrary data through to the configured policy server. Streams the response." + } + }, + { + "name": "Initialize PS Verification", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"documentId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"policyServer\": {}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/services/initializePSVerification", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "initializePSVerification" + ] + }, + "description": "Initialize a policy server verification flow. Streams the response." + } + } + ] + }, + { + "name": "Admin", + "item": [ + { + "name": "Fetch Config", + "request": { + "method": "GET", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/admin/config", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "admin", + "config" + ] + }, + "description": "Fetch the node configuration (admin-only). Note: this GET reads auth fields from the JSON body." + } + }, + { + "name": "Update Config", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"config\": {}\n}" + }, + "url": { + "raw": "{{baseUrl}}/api/admin/config/update", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "admin", + "config", + "update" + ] + }, + "description": "Update the node configuration (admin-only)." + } + } + ] + }, + { + "name": "Queue & Jobs", + "item": [ + { + "name": "Get Index Queue", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/indexQueue", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "indexQueue" + ] + }, + "description": "Get the current indexing queue." + } + }, + { + "name": "Get Jobs for Identifier", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/services/jobs/{{jobId}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "services", + "jobs", + "{{jobId}}" + ] + }, + "description": "Get the indexer job pool for a given job identifier." + } + } + ] + }, + { + "name": "P2P Peers", + "item": [ + { + "name": "Get P2P Network Stats", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/getP2pNetworkStats", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getP2pNetworkStats" + ] + }, + "description": "Get P2P network statistics." + } + }, + { + "name": "Get P2P Peers", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/getP2PPeers", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getP2PPeers" + ] + }, + "description": "List all connected P2P peers." + } + }, + { + "name": "Get P2P Peer", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/getP2PPeer?peerId=", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getP2PPeer" + ], + "query": [ + { + "key": "peerId", + "value": "" + } + ] + }, + "description": "Get details for a specific peer by peerId." + } + }, + { + "name": "Find Peer", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/findPeer?peerId=&timeout=10000", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "findPeer" + ], + "query": [ + { + "key": "peerId", + "value": "" + }, + { + "key": "timeout", + "value": "10000", + "disabled": true + } + ] + }, + "description": "Find a peer's multiaddress by peerId." + } + } + ] + }, + { + "name": "DIDs / Providers", + "item": [ + { + "name": "Get Providers for String", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/getProvidersForString?input={{did}}", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getProvidersForString" + ], + "query": [ + { + "key": "input", + "value": "{{did}}" + } + ] + }, + "description": "Get the providers (peers) that serve a single string/CID." + } + }, + { + "name": "Get Providers for Strings", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "[\n \"did:op:...\",\n \"did:op:...\"\n]" + }, + "url": { + "raw": "{{baseUrl}}/getProvidersForStrings?timeout=10000", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "getProvidersForStrings" + ], + "query": [ + { + "key": "timeout", + "value": "10000", + "disabled": true + } + ] + }, + "description": "Batch lookup: get providers for an array of strings/CIDs. Body is a JSON array of strings." + } + } + ] + }, + { + "name": "Logs", + "item": [ + { + "name": "Get Logs", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/logs?maxLogs=100&page=1", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "logs" + ], + "query": [ + { + "key": "maxLogs", + "value": "100" + }, + { + "key": "page", + "value": "1" + }, + { + "key": "moduleName", + "value": "", + "disabled": true + }, + { + "key": "level", + "value": "", + "disabled": true + }, + { + "key": "startTime", + "value": "", + "disabled": true + }, + { + "key": "endTime", + "value": "", + "disabled": true + } + ] + }, + "description": "Retrieve node logs (admin auth via signed body). Filter/paginate with query params." + } + }, + { + "name": "Get Log by ID", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"address\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"logId\": \"123\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/log/123", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "log", + "123" + ] + }, + "description": "Retrieve a single log entry by id (logId in body must match the :id path param)." + } + } + ] + }, + { + "name": "Direct Command (P2P)", + "description": "POST /directCommand is a single endpoint that dispatches to a protocol command handler chosen by the `command` field. It can run locally or be forwarded to another peer via `node`/`multiAddrs`. Below are representative command bodies; any PROTOCOL_COMMANDS value is accepted (download, encrypt, encryptFile, decryptDDO, getDDO, query, nonce, status, detailedStatus, findDDO, getFees, fileInfo, validateDDO, getComputeEnvironments, startCompute, freeStartCompute, stopCompute, getComputeStatus, getComputeStreamableLogs, getComputeResult, initializeCompute, reindexTx, reindexChain, collectFees, PolicyServerPassthrough, getP2PPeer(s), getP2PNetworkStats, findPeer, createAuthToken, invalidateAuthToken, fetchConfig, pushConfig, getLogs, jobs, persistentStorage*, getAccessList, searchAccessList, getEscrowEvents, serviceGetTemplates, serviceStart, serviceStop, serviceRestart, serviceGetStatus, serviceExtend, serviceGetStreamableLogs, stopNode, ...).", + "item": [ + { + "name": "getDDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"getDDO\",\n \"id\": \"{{did}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Fetch a DDO by id over the command interface." + } + }, + { + "name": "nonce", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"nonce\",\n \"address\": \"{{consumerAddress}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Get the nonce for an address." + } + }, + { + "name": "status", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"status\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Get node status. Use \"command\": \"detailedStatus\" for the extended report." + } + }, + { + "name": "query", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"query\",\n \"query\": { \"query\": { \"match_all\": {} }, \"from\": 0, \"size\": 10 }\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Query indexed assets." + } + }, + { + "name": "getFees", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"getFees\",\n \"ddoId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"validUntil\": 0\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Get the provider fees for an asset service." + } + }, + { + "name": "fileInfo", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"fileInfo\",\n \"did\": \"{{did}}\",\n \"serviceId\": \"\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Get file info for an asset or a raw file descriptor (type/url/...)." + } + }, + { + "name": "download", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"download\",\n \"fileIndex\": 0,\n \"documentId\": \"{{did}}\",\n \"serviceId\": \"\",\n \"transferTxId\": \"\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Download an asset file (streams the response)." + } + }, + { + "name": "encrypt", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"encrypt\",\n \"blob\": \"hello\",\n \"consumerAddress\": \"{{consumerAddress}}\",\n \"signature\": \"{{signature}}\",\n \"nonce\": \"{{nonce}}\",\n \"encoding\": \"string\",\n \"encryptionType\": \"ECIES\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Encrypt a blob (AES or ECIES)." + } + }, + { + "name": "decryptDDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"decryptDDO\",\n \"decrypterAddress\": \"{{consumerAddress}}\",\n \"chainId\": {{chainId}},\n \"nonce\": \"{{nonce}}\",\n \"signature\": \"{{signature}}\",\n \"transactionId\": \"\",\n \"dataNftAddress\": \"\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Decrypt a DDO document." + } + }, + { + "name": "findDDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"findDDO\",\n \"id\": \"{{did}}\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Find which peers can serve a DDO." + } + }, + { + "name": "validateDDO", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"validateDDO\",\n \"ddo\": {}\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Validate a DDO." + } + }, + { + "name": "getComputeEnvironments", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"getComputeEnvironments\",\n \"chainId\": {{chainId}}\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "List compute environments over the command interface." + } + }, + { + "name": "getEscrowEvents", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"command\": \"getEscrowEvents\",\n \"chainId\": {{chainId}}\n}" + }, + "url": { + "raw": "{{baseUrl}}/directCommand", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "directCommand" + ] + }, + "description": "Query escrow events over the command interface." + } + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/services.md b/docs/services.md index 97b5eefcd..3cc2ab548 100644 --- a/docs/services.md +++ b/docs/services.md @@ -28,6 +28,7 @@ and `signature` as query parameters (or an auth-token `Authorization` header). | --- | --- | --- | --- | | `SERVICE_START` | `/api/services/serviceStart` | POST | Validate, persist a `Starting` record, and return the `serviceId` immediately (escrow + image + container happen in the background) | | `SERVICE_GET_STATUS` | `/api/services/serviceStatus` | GET | Read job status / endpoints — authenticated, owner-scoped (see notice below); poll this to follow a starting service | +| `SERVICE_LIST` | `/api/services/serviceList` | GET | Node-wide service listing — authenticated, **not** owner-scoped. Default: only services currently holding a resource reservation; `status=` filters to one specific status, `includeAllStatuses=true` returns everything, `fromTimestamp` keeps services created at/after that moment. Output is listing-sanitized (no `userData`, no `dockerCmd`/`dockerEntrypoint`, no Dockerfile) | | `SERVICE_EXTEND` | `/api/services/serviceExtend` | POST | Pay to push the expiry further out | | `SERVICE_RESTART` | `/api/services/serviceRestart` | POST | Recreate the container (no extra charge); asynchronous like start — returns once the job is `Restarting`, poll `serviceStatus` | | `SERVICE_STOP` | `/api/services/serviceStop` | POST | Tear down the container; the paid resource reservation (cpu/ram/gpu + host ports) is kept until `expiresAt`, so the service can be restarted anytime on the same endpoints | diff --git a/src/@types/commands.ts b/src/@types/commands.ts index d89c6b117..9f87a7e3f 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -450,6 +450,24 @@ export interface ServiceGetStatusCommand extends Command { serviceId?: string } +// Node-wide service listing (SERVICE_LIST), shaped like GetJobsCommand. Authenticated +// like every other service command (consumerAddress + signature/token), but NOT +// owner-scoped: it returns every consumer's services. Default (no filters): only the +// services currently holding a resource reservation. +export interface GetServicesCommand extends Command { + consumerAddress: string + nonce: string + signature: string + // filter to ONE specific status (any ServiceStatusNumber, incl. Expired); takes + // precedence over includeAllStatuses + status?: number + // return services in EVERY status instead of only the resource-holding set + includeAllStatuses?: boolean + // only services created at/after this moment: ISO date string, or a Unix timestamp + // (seconds or milliseconds) as a string + fromTimestamp?: string +} + export interface ServiceRestartCommand extends Command { consumerAddress: string nonce: string diff --git a/src/components/core/handler/coreHandlersRegistry.ts b/src/components/core/handler/coreHandlersRegistry.ts index c1795f8e8..c9c322b25 100644 --- a/src/components/core/handler/coreHandlersRegistry.ts +++ b/src/components/core/handler/coreHandlersRegistry.ts @@ -70,6 +70,7 @@ import { ServiceExtendHandler, ServiceRestartHandler, ServiceGetStatusHandler, + GetServicesHandler, ServiceGetStreamableLogsHandler } from '../service/index.js' @@ -180,6 +181,7 @@ export class CoreHandlersRegistry { PROTOCOL_COMMANDS.SERVICE_GET_STATUS, new ServiceGetStatusHandler(node) ) + this.registerCoreHandler(PROTOCOL_COMMANDS.SERVICE_LIST, new GetServicesHandler(node)) this.registerCoreHandler( PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS, new ServiceGetStreamableLogsHandler(node) diff --git a/src/components/core/service/getServices.ts b/src/components/core/service/getServices.ts new file mode 100644 index 000000000..e2c762afa --- /dev/null +++ b/src/components/core/service/getServices.ts @@ -0,0 +1,118 @@ +import { Readable } from 'stream' +import { P2PCommandResponse } from '../../../@types/index.js' +import { GetServicesCommand } from '../../../@types/commands.js' +import { CommandHandler } from '../handler/handler.js' +import { + ValidateParams, + validateCommandParameters, + buildInvalidRequestMessage +} from '../../httpRoutes/validateCommands.js' +import { + ServiceStatusNumber, + type ServiceJob +} from '../../../@types/C2D/ServiceOnDemand.js' +import { toListedServiceJob } from './utils.js' + +// Parses the `fromTimestamp` filter into Unix milliseconds. Accepts an ISO date string +// or a Unix timestamp (seconds or milliseconds) given as a string / number-like string. +// Returns undefined for "no filter" and null for an unparseable value (caller → 400). +export function parseFromTimestamp(value?: string): number | undefined | null { + if (value === undefined || value === null || value === '') return undefined + if (/^\d+$/.test(String(value))) { + const n = Number(value) + // 1e12 ms ≈ Sep 2001; any plausible seconds value is far below it + return n > 1e12 ? n : n * 1000 + } + const t = Date.parse(String(value)) + return Number.isNaN(t) ? null : t +} + +// SERVICE_LIST: the node-wide service listing, shaped like GetJobsHandler. Default (no +// filters) returns exactly what the engines count against the shared resource pools +// (getRunningServiceJobs): Running/Restarting/Stopping, the mid-start pipeline states, +// paid Error (container died, restartable), and explicitly Stopped (reservation kept +// until expiresAt). `status` narrows to ONE specific status (any, incl. Expired); +// `includeAllStatuses` returns everything; `fromTimestamp` keeps only services created +// at/after that moment. Unlike SERVICE_GET_STATUS this is NOT owner-scoped: any +// authenticated caller sees every consumer's services, so the output is listing-grade +// sanitized (no userData, no CMD/ENTRYPOINT overrides, no Dockerfile). +export class GetServicesHandler extends CommandHandler { + validate(command: GetServicesCommand): ValidateParams { + // consumerAddress is required: it is the identity the signature/token is verified + // against (same contract as the other service commands). + const validation = validateCommandParameters(command, ['consumerAddress']) + if (!validation.valid) return validation + if ( + command.status !== undefined && + ServiceStatusNumber[command.status] === undefined + ) { + return buildInvalidRequestMessage( + `Parameter "status" is not a valid service status number: ${command.status}` + ) + } + if (command.fromTimestamp !== undefined) { + if (typeof command.fromTimestamp !== 'string') + return buildInvalidRequestMessage( + 'Parameter : "fromTimestamp" is not a valid string' + ) + if (parseFromTimestamp(command.fromTimestamp) === null) + return buildInvalidRequestMessage( + `Parameter "fromTimestamp" is not a valid date: "${command.fromTimestamp}" — use an ISO date or a Unix timestamp` + ) + } + return validation + } + + async handle(task: GetServicesCommand): Promise { + const validationResponse = await this.verifyParamsAndRateLimits(task) + if (this.shouldDenyTaskHandling(validationResponse)) return validationResponse + + const auth = await this.validateTokenOrSignature( + task.authorization, + task.consumerAddress, + task.nonce, + task.signature, + task.command + ) + if (auth.status.httpStatus !== 200) return auth + + const engines = this.getOceanNode().getC2DEngines() + if (!engines) + return { + stream: null, + status: { httpStatus: 503, error: 'Compute engines not configured' } + } + + const jobs: ServiceJob[] = [] + for (const eng of engines.getAllEngines()) { + const { hash } = eng.getC2DConfig() + if (task.status !== undefined || task.includeAllStatuses) { + // status-explicit listing: read ALL of this cluster's jobs (getServiceJob has no + // cluster column filter, so scope by clusterHash here), then narrow to the one + // status when given — `status` takes precedence over `includeAllStatuses` + const all = (await eng.db.getServiceJob()).filter( + (j: ServiceJob) => j.clusterHash === hash + ) + jobs.push( + ...(task.status !== undefined + ? all.filter((j: ServiceJob) => j.status === task.status) + : all) + ) + } else { + // default: the cluster's resource-holding set + jobs.push(...(await eng.db.getRunningServiceJobs(hash))) + } + } + + const fromMs = parseFromTimestamp(task.fromTimestamp) + const filtered = + fromMs === undefined || fromMs === null + ? jobs + : jobs.filter((j) => Date.parse(j.dateCreated) >= fromMs) + + return { + stream: Readable.from(JSON.stringify(filtered.map(toListedServiceJob))), + status: { httpStatus: 200 } + } + } +} diff --git a/src/components/core/service/index.ts b/src/components/core/service/index.ts index 842604e5b..b568b2f51 100644 --- a/src/components/core/service/index.ts +++ b/src/components/core/service/index.ts @@ -4,6 +4,7 @@ export { ServiceStopHandler } from './stopService.js' export { ServiceExtendHandler } from './extendService.js' export { ServiceRestartHandler } from './restartService.js' export { ServiceGetStatusHandler } from './getStatus.js' +export { GetServicesHandler } from './getServices.js' export { ServiceGetStreamableLogsHandler } from './getStreamableLogs.js' export * from './utils.js' export * from './templateLoader.js' diff --git a/src/components/core/service/utils.ts b/src/components/core/service/utils.ts index 278db5101..d47221ae6 100644 --- a/src/components/core/service/utils.ts +++ b/src/components/core/service/utils.ts @@ -62,6 +62,30 @@ export function toPublicServiceJob( return pub } +// Listing-grade sanitization for SERVICE_LIST, which is NOT owner-scoped: on top of the +// always-stripped userData (the encrypted env blob), it removes everything that reveals +// HOW a service is configured — CMD/ENTRYPOINT overrides and any inline Dockerfile — +// keeping identity, status, resources, endpoints and payment metadata. The owner-scoped +// SERVICE_GET_STATUS keeps those fields (the owner set them). +export function toListedServiceJob( + job: ServiceJob | null +): Omit< + ServiceJob, + 'userData' | 'dockerCmd' | 'dockerEntrypoint' | 'dockerfile' | 'additionalDockerFiles' +> | null { + if (!job) return null + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { + userData, + dockerCmd, + dockerEntrypoint, + dockerfile, + additionalDockerFiles, + ...pub + } = job + return pub +} + const SINCE_DURATION_RE = /^(\d+)(s|m|h|d)$/ const SINCE_DURATION_UNIT_SECONDS: Record = { s: 1, diff --git a/src/components/httpRoutes/compute.ts b/src/components/httpRoutes/compute.ts index b83c8ac33..d8e68a780 100644 --- a/src/components/httpRoutes/compute.ts +++ b/src/components/httpRoutes/compute.ts @@ -28,6 +28,7 @@ import type { ServiceExtendCommand, ServiceRestartCommand, ServiceGetStatusCommand, + GetServicesCommand, ServiceGetStreamableLogsCommand } from '../../@types/commands.js' import { @@ -37,6 +38,7 @@ import { ServiceExtendHandler, ServiceRestartHandler, ServiceGetStatusHandler, + GetServicesHandler, ServiceGetStreamableLogsHandler } from '../core/service/index.js' @@ -474,6 +476,25 @@ computeRoutes.get(`${SERVICES_API_BASE_PATH}/serviceStatus`, async (req, res) => await runServiceCommand(ServiceGetStatusHandler, task, res) }) +// node-wide service listing (any owner). Default: only services currently holding a +// resource reservation; `status` filters to one specific status, `includeAllStatuses` +// returns everything, `fromTimestamp` keeps services created at/after that moment. +computeRoutes.get(`${SERVICES_API_BASE_PATH}/serviceList`, async (req, res) => { + const task: GetServicesCommand = { + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: req.query.consumerAddress as string, + nonce: req.query.nonce as string, + signature: req.query.signature as string, + status: req.query.status !== undefined ? Number(req.query.status) : undefined, + includeAllStatuses: req.query.includeAllStatuses === 'true', + fromTimestamp: (req.query.fromTimestamp as string) || undefined, + node: (req.query.node as string) || null, + authorization: req.headers?.authorization, + caller: req.caller + } + await runServiceCommand(GetServicesHandler, task, res) +}) + // streaming logs for services computeRoutes.get(`${SERVICES_API_BASE_PATH}/serviceStreamableLogs`, async (req, res) => { try { diff --git a/src/test/integration/services.test.ts b/src/test/integration/services.test.ts index 409081d48..2157ad38b 100644 --- a/src/test/integration/services.test.ts +++ b/src/test/integration/services.test.ts @@ -6,6 +6,7 @@ import { ServiceExtendHandler, ServiceRestartHandler, ServiceGetStatusHandler, + GetServicesHandler, ServiceGetStreamableLogsHandler } from '../../components/core/service/index.js' import { ComputeGetEnvironmentsHandler } from '../../components/core/compute/index.js' @@ -16,6 +17,7 @@ import type { ServiceExtendCommand, ServiceRestartCommand, ServiceGetStatusCommand, + GetServicesCommand, ServiceGetStreamableLogsCommand } from '../../@types/commands.js' import { @@ -486,6 +488,82 @@ describe('********** Service on Demand', () => { expect(unauth.status.httpStatus).to.not.equal(200) }) + it('(e2) SERVICE_LIST returns the node-wide resource-holding set (not owner-scoped)', async () => { + // authenticated as the NON-owner: the running service must still be listed + const { + consumerAddress: addr, + nonce, + signature + } = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) + const resp = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: addr, + nonce, + signature + } as GetServicesCommand) + assert( + resp.status.httpStatus === 200, + `expected 200, got ${resp.status.httpStatus}: ${resp.status?.error ?? ''}` + ) + const jobs = (await streamToObject(resp.stream as Readable)) as ServiceJob[] + const listed = jobs.find((j) => j.serviceId === serviceId) + assert(listed, 'the running service must appear in the node-wide list') + expect(listed.owner.toLowerCase()).to.equal(consumerAddress.toLowerCase()) + // listing-grade sanitization: no env blob, no CMD/ENTRYPOINT overrides + expect((listed as any).userData).to.equal(undefined) + expect((listed as any).dockerCmd).to.equal(undefined) + expect((listed as any).dockerEntrypoint).to.equal(undefined) + + // status filter: Running includes the service, Expired does not + const sig2 = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) + const runningOnly = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: sig2.consumerAddress, + nonce: sig2.nonce, + signature: sig2.signature, + status: ServiceStatusNumber.Running + } as GetServicesCommand) + const runningJobs = (await streamToObject( + runningOnly.stream as Readable + )) as ServiceJob[] + assert( + runningJobs.find((j) => j.serviceId === serviceId), + 'status=Running must include the service' + ) + const sig3 = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) + const expiredOnly = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: sig3.consumerAddress, + nonce: sig3.nonce, + signature: sig3.signature, + status: ServiceStatusNumber.Expired + } as GetServicesCommand) + const expiredJobs = (await streamToObject( + expiredOnly.stream as Readable + )) as ServiceJob[] + expect(expiredJobs.find((j) => j.serviceId === serviceId)).to.equal(undefined) + + // fromTimestamp in the future excludes everything + const sig4 = await signFor(nonOwnerAccount, PROTOCOL_COMMANDS.SERVICE_LIST) + const future = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: sig4.consumerAddress, + nonce: sig4.nonce, + signature: sig4.signature, + includeAllStatuses: true, + fromTimestamp: String(Date.now() + 3600_000) + } as GetServicesCommand) + const futureJobs = (await streamToObject(future.stream as Readable)) as ServiceJob[] + expect(futureJobs.find((j) => j.serviceId === serviceId)).to.equal(undefined) + + // unauthenticated requests are rejected + const unauth = await new GetServicesHandler(oceanNode).handle({ + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: addr + } as GetServicesCommand) + expect(unauth.status.httpStatus).to.not.equal(200) + }) + it('(f) SERVICE_GET_STREAMABLE_LOGS streams the real container logs (owner-only)', async () => { const { consumerAddress: addr, diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index 4cf67bc3d..b06cf5fe5 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -6,6 +6,7 @@ import { PROTOCOL_COMMANDS } from '../../../utils/constants.js' import { ServiceStatusNumber, ServiceJob } from '../../../@types/C2D/ServiceOnDemand.js' import { ServiceGetTemplatesHandler } from '../../../components/core/service/getTemplates.js' import { ServiceGetStatusHandler } from '../../../components/core/service/getStatus.js' +import { GetServicesHandler } from '../../../components/core/service/getServices.js' import { ServiceStartHandler } from '../../../components/core/service/startService.js' import { ServiceStopHandler } from '../../../components/core/service/stopService.js' import { ServiceExtendHandler } from '../../../components/core/service/extendService.js' @@ -93,7 +94,11 @@ function buildFakes(opts: FakeOpts = {}) { ? [opts.serviceJobInDb] : [] ), - updateServiceJob: sinon.stub().resolves(1) + updateServiceJob: sinon.stub().resolves(1), + // SERVICE_LIST: the engine's cluster-scoped resource-holding set + getRunningServiceJobs: sinon + .stub() + .resolves(opts.serviceJobInDb ? [opts.serviceJobInDb] : []) }, escrow, getComputeEnvironments: sinon.stub().resolves([env]), @@ -467,6 +472,127 @@ describe('Service handlers', () => { }) }) + describe('GetServicesHandler (SERVICE_LIST)', () => { + const baseTask = { + command: PROTOCOL_COMMANDS.SERVICE_LIST, + consumerAddress: OWNER, + nonce: '1', + signature: '0xsig' + } + + it('400 when consumerAddress is missing', async () => { + const { node } = buildFakes() + const { consumerAddress, ...noAddress } = baseTask + const res = await new GetServicesHandler(node).handle({ ...noAddress } as any) + expect(res.status.httpStatus).to.equal(400) + }) + + it('400 on an unknown status number or an unparseable fromTimestamp', async () => { + const { node } = buildFakes() + const badStatus = await new GetServicesHandler(node).handle({ + ...baseTask, + status: 1234 + } as any) + expect(badStatus.status.httpStatus).to.equal(400) + const badTs = await new GetServicesHandler(node).handle({ + ...baseTask, + fromTimestamp: 'not-a-date' + } as any) + expect(badTs.status.httpStatus).to.equal(400) + }) + + it('200 default: the node-wide resource-holding set (any owner), listing-sanitized', async () => { + // a service owned by SOMEONE ELSE must be listed too — SERVICE_LIST is not + // owner-scoped like SERVICE_GET_STATUS + const job = makeJob({ + owner: '0xsomeoneelse', + status: ServiceStatusNumber.Stopped, + dockerCmd: ['secret', '--key=abc'], + dockerEntrypoint: ['/entry.sh'], + dockerfile: 'FROM x\nENV SECRET=1' + }) + const { node, engine } = buildFakes({ serviceJobInDb: job }) + const res = await new GetServicesHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + // the engine is queried for ITS cluster's resource-holding set + expect(engine.db.getRunningServiceJobs.calledWith('hash-1')).to.equal(true) + const out = await body(res) + expect(out).to.have.length(1) + expect(out[0].serviceId).to.equal(job.serviceId) + expect(out[0].owner).to.equal('0xsomeoneelse') + expect(out[0].status).to.equal(ServiceStatusNumber.Stopped) + // listing-grade sanitization: no env blob, no CMD/ENTRYPOINT, no Dockerfile + expect(out[0]).to.not.have.property('userData') + expect(out[0]).to.not.have.property('dockerCmd') + expect(out[0]).to.not.have.property('dockerEntrypoint') + expect(out[0]).to.not.have.property('dockerfile') + // identity/resource fields survive + expect(out[0].resources).to.deep.equal(job.resources) + expect(out[0].endpoints).to.deep.equal(job.endpoints) + }) + + it('status filter: returns only that status (incl. Expired, outside the resource set)', async () => { + const expired = makeJob({ status: ServiceStatusNumber.Expired }) + const { node, engine } = buildFakes({ serviceJobInDb: expired }) + const res = await new GetServicesHandler(node).handle({ + ...baseTask, + status: ServiceStatusNumber.Expired + } as any) + expect(res.status.httpStatus).to.equal(200) + // status-explicit listing reads ALL jobs, not the resource-holding query + expect(engine.db.getRunningServiceJobs.called).to.equal(false) + const out = await body(res) + expect(out).to.have.length(1) + expect(out[0].status).to.equal(ServiceStatusNumber.Expired) + + // a status nothing matches → empty list + const none = await new GetServicesHandler(node).handle({ + ...baseTask, + status: ServiceStatusNumber.Running + } as any) + expect(await body(none)).to.deep.equal([]) + }) + + it('includeAllStatuses: returns every status; fromTimestamp narrows by creation date', async () => { + const job = makeJob({ + status: ServiceStatusNumber.Expired, + dateCreated: new Date('2026-01-15T00:00:00Z').toISOString() + }) + const { node } = buildFakes({ serviceJobInDb: job }) + const all = await new GetServicesHandler(node).handle({ + ...baseTask, + includeAllStatuses: true + } as any) + expect(await body(all)).to.have.length(1) + + // created before the cutoff → filtered out (ISO form) + const afterIso = await new GetServicesHandler(node).handle({ + ...baseTask, + includeAllStatuses: true, + fromTimestamp: '2026-02-01T00:00:00Z' + } as any) + expect(await body(afterIso)).to.deep.equal([]) + + // created after the cutoff → kept (Unix-seconds form) + const beforeUnix = await new GetServicesHandler(node).handle({ + ...baseTask, + includeAllStatuses: true, + fromTimestamp: String( + Math.floor(new Date('2026-01-01T00:00:00Z').getTime() / 1000) + ) + } as any) + expect(await body(beforeUnix)).to.have.length(1) + }) + + it('200 with an empty list when nothing holds resources', async () => { + const { node } = buildFakes({ serviceJobInDb: null }) + const res = await new GetServicesHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(200) + const out = await body(res) + expect(out).to.deep.equal([]) + }) + }) + describe('ServiceStartHandler', () => { const baseTask = { command: PROTOCOL_COMMANDS.SERVICE_START, diff --git a/src/utils/constants.ts b/src/utils/constants.ts index 178e9395d..507b1b1bb 100644 --- a/src/utils/constants.ts +++ b/src/utils/constants.ts @@ -56,6 +56,7 @@ export const PROTOCOL_COMMANDS = { SERVICE_STOP: 'serviceStop', SERVICE_RESTART: 'serviceRestart', SERVICE_GET_STATUS: 'serviceGetStatus', + SERVICE_LIST: 'serviceList', SERVICE_EXTEND: 'serviceExtend', SERVICE_GET_STREAMABLE_LOGS: 'serviceGetStreamableLogs' } @@ -115,6 +116,7 @@ export const SUPPORTED_PROTOCOL_COMMANDS: string[] = [ PROTOCOL_COMMANDS.SERVICE_STOP, PROTOCOL_COMMANDS.SERVICE_RESTART, PROTOCOL_COMMANDS.SERVICE_GET_STATUS, + PROTOCOL_COMMANDS.SERVICE_LIST, PROTOCOL_COMMANDS.SERVICE_EXTEND, PROTOCOL_COMMANDS.SERVICE_GET_STREAMABLE_LOGS ] From b884319974bb1928f05bec1c8abb3abf6113e5d7 Mon Sep 17 00:00:00 2001 From: alexcos20 Date: Fri, 17 Jul 2026 07:49:19 +0300 Subject: [PATCH 10/10] fix reviews --- src/components/c2d/compute_engine_docker.ts | 13 ++++-- src/components/core/service/extendService.ts | 40 ++++++++++++++++++- src/test/unit/compute.test.ts | 5 ++- src/test/unit/service/serviceHandlers.test.ts | 22 ++++++++++ .../service/serviceNetworkCleanup.test.ts | 5 ++- .../unit/service/serviceRestartRace.test.ts | 27 +++++++++++++ 6 files changed, 105 insertions(+), 7 deletions(-) diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts index c3a437976..b930751e8 100755 --- a/src/components/c2d/compute_engine_docker.ts +++ b/src/components/c2d/compute_engine_docker.ts @@ -3900,12 +3900,17 @@ export class C2DEngineDocker extends C2DEngine { SERVICE_LOCK_STALE_MS ) } catch (e: any) { - // DB trouble must not brick every lifecycle op on a single-process node — fall - // back to in-process-only locking (the pre-lease behavior). + // Fail CLOSED: proceeding without the SQLite lease would disable cross-process + // exclusion exactly when it matters most — SQLITE_BUSY-style errors are most + // likely precisely when two processes contend. Deferring costs little: every + // lifecycle operation writes to the same SQLite file moments later anyway, so it + // could not complete with a broken DB regardless. Callers reject with "retry + // shortly" (handlers) or retry on the next tick (the loop). CORE_LOGGER.warn( - `service lock DB acquire failed for ${serviceId} (falling back to in-process lock): ${e.message}` + `service lock DB acquire failed for ${serviceId} — deferring the operation (fail closed): ${e.message}` ) - acquired = true + this.serviceOpsInFlight.delete(serviceId) + return false } if (!acquired) { this.serviceOpsInFlight.delete(serviceId) diff --git a/src/components/core/service/extendService.ts b/src/components/core/service/extendService.ts index 33ee1aef4..ec34e71ad 100644 --- a/src/components/core/service/extendService.ts +++ b/src/components/core/service/extendService.ts @@ -251,7 +251,45 @@ export class ServiceExtendHandler extends CommandHandler { cost: costExtend } freshJob.extendPayments = [...(freshJob.extendPayments ?? []), intent] - await engine.db.updateServiceJob(freshJob) + try { + await engine.db.updateServiceJob(freshJob) + } catch (persistErr: any) { + // The lock is mined but we could NOT make it durable — without a record, the + // unresolved-intent recovery would never find it. Compensate: refund the + // lock now. If even that fails, funds are stranded on-chain → 409 so a + // human reconciles; nothing was claimed either way. + CORE_LOGGER.error( + `Service extend: intent persistence failed (${persistErr.message}) — refunding lock ${lockTx}` + ) + freshJob.extendPayments = freshJob.extendPayments.filter((p) => p !== intent) + const cancelTx = await engine.escrow + .cancelExpiredLock( + task.payment.chainId, + task.serviceId, + task.payment.token, + task.consumerAddress + ) + .catch((e): string | null => { + CORE_LOGGER.error(`cancelExpiredLock failed: ${e.message}`) + return null + }) + if (!cancelTx) + return { + stream: null, + status: { + httpStatus: 409, + error: `Extension aborted: the payment intent could not be persisted AND the escrow lock ${lockTx} could not be refunded — contact the node operator` + } + } + return { + stream: null, + status: { + httpStatus: 402, + error: + 'Extension aborted before charging: the payment intent could not be persisted — the escrow lock was refunded' + } + } + } let claimTx: string | null try { diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 90412598a..398123c2a 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -1529,7 +1529,10 @@ describe('service start/restart Docker cleanup on failure', function () { engine.db = { newServiceJob: sinon.stub().resolves(), - updateServiceJob: sinon.stub().resolves() + updateServiceJob: sinon.stub().resolves(), + // lifecycle lease (fail-closed: restart/stop reject without it) + acquireServiceLock: sinon.stub().resolves(true), + releaseServiceLock: sinon.stub().resolves(undefined) } engine.getC2DConfig = sinon.stub().returns({ hash: 'cluster-hash', diff --git a/src/test/unit/service/serviceHandlers.test.ts b/src/test/unit/service/serviceHandlers.test.ts index b06cf5fe5..2a4f8812e 100644 --- a/src/test/unit/service/serviceHandlers.test.ts +++ b/src/test/unit/service/serviceHandlers.test.ts @@ -449,6 +449,28 @@ describe('Service handlers', () => { expect(out[0].extendPayments[1].claimTx).to.equal('0xclaim') }) + it('402 and refunds the lock when the intent cannot be persisted (no unrecorded charge)', async () => { + const { node, escrow, engine } = buildFakes({ serviceJobInDb: makeJob() }) + // first write in the flow is the durable intent — make it fail + engine.db.updateServiceJob.onFirstCall().rejects(new Error('db down')) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(402) + expect(String(res.status.error)).to.contain('refunded') + // the mined lock was compensated, and the claim was never attempted + expect(escrow.cancelExpiredLock.calledOnce).to.equal(true) + expect(escrow.claimLock.called).to.equal(false) + }) + + it('409 when the intent cannot be persisted AND the lock refund fails (stranded funds)', async () => { + const { node, escrow, engine } = buildFakes({ serviceJobInDb: makeJob() }) + engine.db.updateServiceJob.onFirstCall().rejects(new Error('db down')) + escrow.cancelExpiredLock.rejects(new Error('rpc down')) + const res = await new ServiceExtendHandler(node).handle({ ...baseTask } as any) + expect(res.status.httpStatus).to.equal(409) + expect(String(res.status.error)).to.contain('could not be refunded') + expect(escrow.claimLock.called).to.equal(false) + }) + it('409 when an unresolved extension intent cannot be refunded (no double charge)', async () => { const job = makeJob({ extendPayments: [ diff --git a/src/test/unit/service/serviceNetworkCleanup.test.ts b/src/test/unit/service/serviceNetworkCleanup.test.ts index 70e4d15cb..db1d59e95 100644 --- a/src/test/unit/service/serviceNetworkCleanup.test.ts +++ b/src/test/unit/service/serviceNetworkCleanup.test.ts @@ -210,7 +210,10 @@ describe('service network cleanup (leaked ocean-svc- networks)', () => { const job = makeJob({ containerId: '', networkId: '' }) engine.db = { getServiceJob: sinon.stub().resolves([job]), - updateServiceJob: sinon.stub().resolves(1) + updateServiceJob: sinon.stub().resolves(1), + // lifecycle lease (fail-closed: stopService rejects without it) + acquireServiceLock: sinon.stub().resolves(true), + releaseServiceLock: sinon.stub().resolves(undefined) } const network = fakeNetwork({}) engine.docker.getNetwork.returns(network) diff --git a/src/test/unit/service/serviceRestartRace.test.ts b/src/test/unit/service/serviceRestartRace.test.ts index 7917075b8..f8bc6104e 100644 --- a/src/test/unit/service/serviceRestartRace.test.ts +++ b/src/test/unit/service/serviceRestartRace.test.ts @@ -363,6 +363,33 @@ describe('service lifecycle lock (restart/stop vs InternalLoop races)', () => { expect(engine.db.getServiceJob.called).to.equal(false) }) + it('lease acquisition fails CLOSED when the lock DB errors (no docker/DB mutations)', async () => { + const engine = makeEngine() + engine.db.acquireServiceLock = sinon.stub().rejects(new Error('SQLITE_BUSY')) + + try { + await engine.restartService(SERVICE_ID, OWNER) + expect.fail('expected restartService to reject') + } catch (e: any) { + expect(e.message).to.contain('operation in progress') + } + // rolled back, nothing touched — the operation is deferred, not run lock-less + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + expect(engine.db.getServiceJob.called).to.equal(false) + expect(engine.db.updateServiceJob.called).to.equal(false) + expect(engine.docker.getContainer.called).to.equal(false) + + // the loop's tryAcquire path defers too (skips the pending job this tick) + engine.db.getPendingServiceStarts.resolves([ + makeJob({ status: ServiceStatusNumber.PullImage, statusText: 'PullImage' }) + ]) + engine.processServiceStart = sinon.stub().resolves() + await engine.InternalLoop() + await flush() + expect(engine.processServiceStart.called).to.equal(false) + expect(engine.serviceOpsInFlight.has(SERVICE_ID)).to.equal(false) + }) + it('InternalLoop skips a pending job whose DB lease is held by another process', async () => { const engine = makeEngine() engine.db.getPendingServiceStarts.resolves([