From 0fa92aee142d774d0749de4c8ae7bda2fe5520e2 Mon Sep 17 00:00:00 2001 From: Frank Rousseau Date: Tue, 7 Jul 2026 12:02:43 +0200 Subject: [PATCH] [browsing] Consume the with-tasks routes as compact NDJSON streams Assets and shots full-project views now request stream=true and compact=true: the server streams one entity per line as positional arrays described by the header field lists, which halves the payload and keeps the response out of the server memory. Rows are mapped back by field name (never by position) in the API client, so the stores keep receiving the same list of dicts. Any non-NDJSON answer (older Zou, proxy) falls back to the legacy plain JSON request. --- src/store/api/assets.js | 2 +- src/store/api/client.js | 80 ++++++++++++++++++++++++++ src/store/api/shots.js | 2 +- tests/unit/store/api/client.spec.js | 87 +++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 2 deletions(-) diff --git a/src/store/api/assets.js b/src/store/api/assets.js index cb3787c817..2812bc726f 100644 --- a/src/store/api/assets.js +++ b/src/store/api/assets.js @@ -12,7 +12,7 @@ export default { if (episode) { path += `&episode_id=${episode.id}` } - return client.pget(path) + return withTasks ? client.pgetNdjson(path) : client.pget(path) }, getSharedAssets(production, shared = true) { diff --git a/src/store/api/client.js b/src/store/api/client.js index 30344e0ec0..5164a17e12 100644 --- a/src/store/api/client.js +++ b/src/store/api/client.js @@ -20,6 +20,55 @@ function handleError(err) { // File uploads (ppostFile) stay unbounded — multi-GB movies are legit. const REQUEST_TIMEOUT = { response: 60000, deadline: 300000 } +// Compact rows are positional arrays described by the field lists sent +// in the NDJSON header: always map by name, never by position. +function decodeCompactRow(row, fields, taskFields) { + const entity = {} + fields.forEach((field, index) => { + entity[field] = + field === 'tasks' + ? row[index].map(task => decodeCompactRow(task, taskFields)) + : row[index] + }) + return entity +} + +async function handleNdjsonResponse(response) { + let header = null + let entityFields = null + const entities = [] + const handleLine = line => { + if (!line) return + if (!header) { + header = JSON.parse(line) + const fieldsKey = Object.keys(header).find( + key => key.endsWith('_fields') && key !== 'task_fields' + ) + entityFields = header[fieldsKey] + return + } + const row = JSON.parse(line) + entities.push( + header.compact + ? decodeCompactRow(row, entityFields, header.task_fields) + : row + ) + } + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n') + buffer = lines.pop() + lines.forEach(handleLine) + } + handleLine((buffer + decoder.decode()).trim()) + return entities +} + const client = { request(method, path, data) { return superagent(method, path) @@ -33,6 +82,37 @@ const client = { return client.request('GET', path) }, + // Kitsu-only mode of the with-tasks routes: the server streams NDJSON + // (a header line, then one entity per line as compact positional + // arrays), which halves the payload and keeps the full response out + // of the server memory. Any answer that is not NDJSON — older Zou + // rejecting the parameters, proxy in between — falls back to the + // legacy plain JSON request. + pgetNdjson(path) { + const separator = path.includes('?') ? '&' : '?' + const streamPath = `${path}${separator}stream=true&compact=true` + return fetch(streamPath, { + headers: { Accept: 'application/x-ndjson' }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT.deadline) + }).then( + response => { + if (response.status === 401) { + errors.backToLogin() + // Freeze the chain until the redirect happens. + return new Promise(() => {}) + } + const contentType = response.headers.get('Content-Type') || '' + if (!response.ok || !contentType.includes('ndjson')) { + // Real HTTP errors (403…) are re-raised by the legacy request + // with the error shape the stores already handle. + return client.pget(path) + } + return handleNdjsonResponse(response) + }, + () => client.pget(path) + ) + }, + ppost(path, data) { return client.request('POST', path, data) }, diff --git a/src/store/api/shots.js b/src/store/api/shots.js index 4a0b3a67a0..910afb8c56 100644 --- a/src/store/api/shots.js +++ b/src/store/api/shots.js @@ -9,7 +9,7 @@ export default { let path = '/api/data/shots/with-tasks' if (production) path += `?project_id=${production.id}` if (episode) path += `&episode_id=${episode.id}` - return client.pget(path) + return client.pgetNdjson(path) }, getSequence(sequenceId) { diff --git a/tests/unit/store/api/client.spec.js b/tests/unit/store/api/client.spec.js index d379b44321..3e412db8d2 100644 --- a/tests/unit/store/api/client.spec.js +++ b/tests/unit/store/api/client.spec.js @@ -147,4 +147,91 @@ describe('store/api/client', () => { await client.searchData('rabbit', 10, 0, ['assets'], 'production-1') expect(h.sent.project_id).toEqual('production-1') }) + + describe('pgetNdjson', () => { + // NDJSON body split in two chunks, cutting a line in half, to + // exercise the buffering. + const ndjsonResponse = lines => { + const text = lines.map(line => JSON.stringify(line)).join('\n') + '\n' + const encoder = new TextEncoder() + const middle = Math.floor(text.length / 2) + return { + ok: true, + status: 200, + headers: { get: () => 'application/x-ndjson' }, + body: new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(text.slice(0, middle))) + controller.enqueue(encoder.encode(text.slice(middle))) + controller.close() + } + }) + } + } + + afterEach(() => { + vi.unstubAllGlobals() + }) + + test('requests stream+compact and maps rows by field name', async () => { + const fetchMock = vi.fn(async () => + ndjsonResponse([ + { + compact: true, + asset_fields: ['id', 'name', 'tasks'], + task_fields: ['id', 'task_status_id'] + }, + ['asset-1', 'Chair', [['task-1', 'status-open']]], + ['asset-2', 'Table', []] + ]) + ) + vi.stubGlobal('fetch', fetchMock) + const entities = await client.pgetNdjson( + '/api/data/assets/with-tasks?project_id=production-1' + ) + expect(fetchMock.mock.calls[0][0]).toEqual( + '/api/data/assets/with-tasks?project_id=production-1' + + '&stream=true&compact=true' + ) + expect(entities).toEqual([ + { + id: 'asset-1', + name: 'Chair', + tasks: [{ id: 'task-1', task_status_id: 'status-open' }] + }, + { id: 'asset-2', name: 'Table', tasks: [] } + ]) + }) + + test('falls back to the legacy request when the answer is not NDJSON', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: false, + status: 400, + headers: { get: () => 'application/json' } + })) + ) + h.response = { body: [{ id: 'asset-1' }] } + await expect( + client.pgetNdjson('/api/data/shots/with-tasks') + ).resolves.toEqual([{ id: 'asset-1' }]) + expect(h.calls).toEqual([ + { method: 'GET', path: '/api/data/shots/with-tasks' } + ]) + }) + + test('a 401 redirects to login and never settles the promise', async () => { + vi.stubGlobal('fetch', vi.fn(async () => ({ status: 401 }))) + const outcome = await Promise.race([ + client.pgetNdjson('/api/data/shots/with-tasks').then( + () => 'resolved', + () => 'rejected' + ), + new Promise(resolve => setTimeout(() => resolve('pending'), 20)) + ]) + expect(outcome).toBe('pending') + expect(errors.backToLogin).toHaveBeenCalled() + }) + }) })