[browsing] Consume the with-tasks routes as compact NDJSON streams

src/store/api/assets.js

@@ -12,7 +12,7 @@ export default {
     if (episode) {
       path += `&episode_id=${episode.id}`
     }
-    return client.pget(path)
+    return withTasks ? client.pgetNdjson(path) : client.pget(path)
   },
 
   getSharedAssets(production, shared = true) {

src/store/api/client.js

@@ -20,6 +20,55 @@ function handleError(err) {
 // File uploads (ppostFile) stay unbounded — multi-GB movies are legit.
 const REQUEST_TIMEOUT = { response: 60000, deadline: 300000 }
 
+// Compact rows are positional arrays described by the field lists sent
+// in the NDJSON header: always map by name, never by position.
+function decodeCompactRow(row, fields, taskFields) {
+  const entity = {}
+  fields.forEach((field, index) => {
+    entity[field] =
+      field === 'tasks'
+        ? row[index].map(task => decodeCompactRow(task, taskFields))
+        : row[index]
+  })
+  return entity
+}
+
+async function handleNdjsonResponse(response) {
+  let header = null
+  let entityFields = null
+  const entities = []
+  const handleLine = line => {
+    if (!line) return
+    if (!header) {
+      header = JSON.parse(line)
+      const fieldsKey = Object.keys(header).find(
+        key => key.endsWith('_fields') && key !== 'task_fields'
+      )
+      entityFields = header[fieldsKey]
+      return
+    }
+    const row = JSON.parse(line)
+    entities.push(
+      header.compact
+        ? decodeCompactRow(row, entityFields, header.task_fields)
+        : row
+    )
+  }
+  const reader = response.body.getReader()
+  const decoder = new TextDecoder()
+  let buffer = ''
+  for (;;) {
+    const { done, value } = await reader.read()
+    if (done) break
+    buffer += decoder.decode(value, { stream: true })
+    const lines = buffer.split('\n')
+    buffer = lines.pop()
+    lines.forEach(handleLine)
+  }
+  handleLine((buffer + decoder.decode()).trim())
+  return entities
+}
+
 const client = {
   request(method, path, data) {
     return superagent(method, path)
@@ -33,6 +82,37 @@ const client = {
     return client.request('GET', path)
   },
 
+  // Kitsu-only mode of the with-tasks routes: the server streams NDJSON
+  // (a header line, then one entity per line as compact positional
+  // arrays), which halves the payload and keeps the full response out
+  // of the server memory. Any answer that is not NDJSON — older Zou
+  // rejecting the parameters, proxy in between — falls back to the
+  // legacy plain JSON request.
+  pgetNdjson(path) {
+    const separator = path.includes('?') ? '&' : '?'
+    const streamPath = `${path}${separator}stream=true&compact=true`
+    return fetch(streamPath, {
+      headers: { Accept: 'application/x-ndjson' },
+      signal: AbortSignal.timeout(REQUEST_TIMEOUT.deadline)
+    }).then(
+      response => {
+        if (response.status === 401) {
+          errors.backToLogin()
+          // Freeze the chain until the redirect happens.
+          return new Promise(() => {})
+        }
+        const contentType = response.headers.get('Content-Type') || ''
+        if (!response.ok || !contentType.includes('ndjson')) {
+          // Real HTTP errors (403…) are re-raised by the legacy request
+          // with the error shape the stores already handle.
+          return client.pget(path)
+        }
+        return handleNdjsonResponse(response)
+      },
+      () => client.pget(path)
+    )
+  },
+
   ppost(path, data) {
     return client.request('POST', path, data)
   },

src/store/api/shots.js

@@ -9,7 +9,7 @@ export default {
     let path = '/api/data/shots/with-tasks'
     if (production) path += `?project_id=${production.id}`
     if (episode) path += `&episode_id=${episode.id}`
-    return client.pget(path)
+    return client.pgetNdjson(path)
   },
 
   getSequence(sequenceId) {

tests/unit/store/api/client.spec.js

@@ -147,4 +147,91 @@ describe('store/api/client', () => {
     await client.searchData('rabbit', 10, 0, ['assets'], 'production-1')
     expect(h.sent.project_id).toEqual('production-1')
   })
+
+  describe('pgetNdjson', () => {
+    // NDJSON body split in two chunks, cutting a line in half, to
+    // exercise the buffering.
+    const ndjsonResponse = lines => {
+      const text = lines.map(line => JSON.stringify(line)).join('\n') + '\n'
+      const encoder = new TextEncoder()
+      const middle = Math.floor(text.length / 2)
+      return {
+        ok: true,
+        status: 200,
+        headers: { get: () => 'application/x-ndjson' },
+        body: new ReadableStream({
+          start(controller) {
+            controller.enqueue(encoder.encode(text.slice(0, middle)))
+            controller.enqueue(encoder.encode(text.slice(middle)))
+            controller.close()
+          }
+        })
+      }
+    }
+
+    afterEach(() => {
+      vi.unstubAllGlobals()
+    })
+
+    test('requests stream+compact and maps rows by field name', async () => {
+      const fetchMock = vi.fn(async () =>
+        ndjsonResponse([
+          {
+            compact: true,
+            asset_fields: ['id', 'name', 'tasks'],
+            task_fields: ['id', 'task_status_id']
+          },
+          ['asset-1', 'Chair', [['task-1', 'status-open']]],
+          ['asset-2', 'Table', []]
+        ])
+      )
+      vi.stubGlobal('fetch', fetchMock)
+      const entities = await client.pgetNdjson(
+        '/api/data/assets/with-tasks?project_id=production-1'
+      )
+      expect(fetchMock.mock.calls[0][0]).toEqual(
+        '/api/data/assets/with-tasks?project_id=production-1' +
+          '&stream=true&compact=true'
+      )
+      expect(entities).toEqual([
+        {
+          id: 'asset-1',
+          name: 'Chair',
+          tasks: [{ id: 'task-1', task_status_id: 'status-open' }]
+        },
+        { id: 'asset-2', name: 'Table', tasks: [] }
+      ])
+    })
+
+    test('falls back to the legacy request when the answer is not NDJSON', async () => {
+      vi.stubGlobal(
+        'fetch',
+        vi.fn(async () => ({
+          ok: false,
+          status: 400,
+          headers: { get: () => 'application/json' }
+        }))
+      )
+      h.response = { body: [{ id: 'asset-1' }] }
+      await expect(
+        client.pgetNdjson('/api/data/shots/with-tasks')
+      ).resolves.toEqual([{ id: 'asset-1' }])
+      expect(h.calls).toEqual([
+        { method: 'GET', path: '/api/data/shots/with-tasks' }
+      ])
+    })
+
+    test('a 401 redirects to login and never settles the promise', async () => {
+      vi.stubGlobal('fetch', vi.fn(async () => ({ status: 401 })))
+      const outcome = await Promise.race([
+        client.pgetNdjson('/api/data/shots/with-tasks').then(
+          () => 'resolved',
+          () => 'rejected'
+        ),
+        new Promise(resolve => setTimeout(() => resolve('pending'), 20))
+      ])
+      expect(outcome).toBe('pending')
+      expect(errors.backToLogin).toHaveBeenCalled()
+    })
+  })
 })