diff --git a/src/store/api/assets.js b/src/store/api/assets.js index cb3787c817..2812bc726f 100644 --- a/src/store/api/assets.js +++ b/src/store/api/assets.js @@ -12,7 +12,7 @@ export default { if (episode) { path += `&episode_id=${episode.id}` } - return client.pget(path) + return withTasks ? client.pgetNdjson(path) : client.pget(path) }, getSharedAssets(production, shared = true) { diff --git a/src/store/api/client.js b/src/store/api/client.js index 30344e0ec0..5164a17e12 100644 --- a/src/store/api/client.js +++ b/src/store/api/client.js @@ -20,6 +20,55 @@ function handleError(err) { // File uploads (ppostFile) stay unbounded — multi-GB movies are legit. const REQUEST_TIMEOUT = { response: 60000, deadline: 300000 } +// Compact rows are positional arrays described by the field lists sent +// in the NDJSON header: always map by name, never by position. +function decodeCompactRow(row, fields, taskFields) { + const entity = {} + fields.forEach((field, index) => { + entity[field] = + field === 'tasks' + ? row[index].map(task => decodeCompactRow(task, taskFields)) + : row[index] + }) + return entity +} + +async function handleNdjsonResponse(response) { + let header = null + let entityFields = null + const entities = [] + const handleLine = line => { + if (!line) return + if (!header) { + header = JSON.parse(line) + const fieldsKey = Object.keys(header).find( + key => key.endsWith('_fields') && key !== 'task_fields' + ) + entityFields = header[fieldsKey] + return + } + const row = JSON.parse(line) + entities.push( + header.compact + ? decodeCompactRow(row, entityFields, header.task_fields) + : row + ) + } + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n') + buffer = lines.pop() + lines.forEach(handleLine) + } + handleLine((buffer + decoder.decode()).trim()) + return entities +} + const client = { request(method, path, data) { return superagent(method, path) @@ -33,6 +82,37 @@ const client = { return client.request('GET', path) }, + // Kitsu-only mode of the with-tasks routes: the server streams NDJSON + // (a header line, then one entity per line as compact positional + // arrays), which halves the payload and keeps the full response out + // of the server memory. Any answer that is not NDJSON — older Zou + // rejecting the parameters, proxy in between — falls back to the + // legacy plain JSON request. + pgetNdjson(path) { + const separator = path.includes('?') ? '&' : '?' + const streamPath = `${path}${separator}stream=true&compact=true` + return fetch(streamPath, { + headers: { Accept: 'application/x-ndjson' }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT.deadline) + }).then( + response => { + if (response.status === 401) { + errors.backToLogin() + // Freeze the chain until the redirect happens. + return new Promise(() => {}) + } + const contentType = response.headers.get('Content-Type') || '' + if (!response.ok || !contentType.includes('ndjson')) { + // Real HTTP errors (403…) are re-raised by the legacy request + // with the error shape the stores already handle. + return client.pget(path) + } + return handleNdjsonResponse(response) + }, + () => client.pget(path) + ) + }, + ppost(path, data) { return client.request('POST', path, data) }, diff --git a/src/store/api/shots.js b/src/store/api/shots.js index 4a0b3a67a0..910afb8c56 100644 --- a/src/store/api/shots.js +++ b/src/store/api/shots.js @@ -9,7 +9,7 @@ export default { let path = '/api/data/shots/with-tasks' if (production) path += `?project_id=${production.id}` if (episode) path += `&episode_id=${episode.id}` - return client.pget(path) + return client.pgetNdjson(path) }, getSequence(sequenceId) { diff --git a/tests/unit/store/api/client.spec.js b/tests/unit/store/api/client.spec.js index d379b44321..3e412db8d2 100644 --- a/tests/unit/store/api/client.spec.js +++ b/tests/unit/store/api/client.spec.js @@ -147,4 +147,91 @@ describe('store/api/client', () => { await client.searchData('rabbit', 10, 0, ['assets'], 'production-1') expect(h.sent.project_id).toEqual('production-1') }) + + describe('pgetNdjson', () => { + // NDJSON body split in two chunks, cutting a line in half, to + // exercise the buffering. + const ndjsonResponse = lines => { + const text = lines.map(line => JSON.stringify(line)).join('\n') + '\n' + const encoder = new TextEncoder() + const middle = Math.floor(text.length / 2) + return { + ok: true, + status: 200, + headers: { get: () => 'application/x-ndjson' }, + body: new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(text.slice(0, middle))) + controller.enqueue(encoder.encode(text.slice(middle))) + controller.close() + } + }) + } + } + + afterEach(() => { + vi.unstubAllGlobals() + }) + + test('requests stream+compact and maps rows by field name', async () => { + const fetchMock = vi.fn(async () => + ndjsonResponse([ + { + compact: true, + asset_fields: ['id', 'name', 'tasks'], + task_fields: ['id', 'task_status_id'] + }, + ['asset-1', 'Chair', [['task-1', 'status-open']]], + ['asset-2', 'Table', []] + ]) + ) + vi.stubGlobal('fetch', fetchMock) + const entities = await client.pgetNdjson( + '/api/data/assets/with-tasks?project_id=production-1' + ) + expect(fetchMock.mock.calls[0][0]).toEqual( + '/api/data/assets/with-tasks?project_id=production-1' + + '&stream=true&compact=true' + ) + expect(entities).toEqual([ + { + id: 'asset-1', + name: 'Chair', + tasks: [{ id: 'task-1', task_status_id: 'status-open' }] + }, + { id: 'asset-2', name: 'Table', tasks: [] } + ]) + }) + + test('falls back to the legacy request when the answer is not NDJSON', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: false, + status: 400, + headers: { get: () => 'application/json' } + })) + ) + h.response = { body: [{ id: 'asset-1' }] } + await expect( + client.pgetNdjson('/api/data/shots/with-tasks') + ).resolves.toEqual([{ id: 'asset-1' }]) + expect(h.calls).toEqual([ + { method: 'GET', path: '/api/data/shots/with-tasks' } + ]) + }) + + test('a 401 redirects to login and never settles the promise', async () => { + vi.stubGlobal('fetch', vi.fn(async () => ({ status: 401 }))) + const outcome = await Promise.race([ + client.pgetNdjson('/api/data/shots/with-tasks').then( + () => 'resolved', + () => 'rejected' + ), + new Promise(resolve => setTimeout(() => resolve('pending'), 20)) + ]) + expect(outcome).toBe('pending') + expect(errors.backToLogin).toHaveBeenCalled() + }) + }) })