Skip to content

Commit 5fc5ec5

Browse files
authored
Merge pull request #621 from cipherstash/feat/eql-v3-json
feat(stack): EQL v3 JSON support — types.Json + containment
2 parents 7c10347 + f4d45dd commit 5fc5ec5

22 files changed

Lines changed: 678 additions & 32 deletions

‎.changeset/eql-v3-json-skills.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
'stash': patch
3+
---
4+
5+
Document EQL v3 JSON columns in the bundled skills: `types.Json` in the
6+
`stash-encryption` typed-schema catalog (capability suffix, family, and an
7+
encrypted-JSONB query section), and `contains(col, subObject)` JSON containment
8+
on the v3 Drizzle operators in `stash-drizzle`.

‎.changeset/eql-v3-json.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
'@cipherstash/stack': minor
3+
---
4+
5+
Add EQL v3 JSON columns. `types.Json('col')` declares a `public.eql_v3_json`
6+
column that encrypts a JSON document to an ste_vec `SteVecDocument` and
7+
round-trips it losslessly through `encrypt`/`decrypt` and the model path. A new
8+
`searchableJson` query capability emits the ste_vec index; the index uses
9+
`mode: 'compat'`, which eql-3.0.0's `eql_v3_json` requires (it orders ste_vec
10+
entries by the CLLW-OPE `op` term, so v2's `'standard'`/CLLW-`oc` terms are
11+
rejected).
12+
13+
The Drizzle integration's `contains(col, subObject)` now answers encrypted-JSONB
14+
containment on a `types.Json` column, emitting the `@>` operator with a
15+
`query_jsonb` needle (from `encryptQuery`). The ste_vec index indexes array
16+
elements by identity but not position, so containment is a true subset test
17+
(`{ roles: ['x'] }` matches any document whose `roles` array contains `x`,
18+
regardless of index).

‎packages/stack/__tests__/drizzle-v3/operators.test.ts‎

Lines changed: 94 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,21 @@ function setup(
5050
}),
5151
) {
5252
const encrypt = vi.fn((...args: unknown[]) => chainable(encryptImpl(...args)))
53+
// `encryptQuery` backs JSON containment: it returns a narrowed `query_jsonb`
54+
// term (no ciphertext). The double returns a recognisable ste_vec-shaped term
55+
// so a rendered `@>` query can be asserted.
56+
const encryptQuery = vi.fn(() =>
57+
chainable(
58+
Promise.resolve({ data: { sv: [{ s: 'sel', hm: 'h' }] } }) as never,
59+
),
60+
)
5361
// The factory's `client` parameter is the structural `{ encrypt }` surface,
5462
// so this hand-rolled double satisfies it with no cast (M1).
55-
const client = { encrypt }
63+
const client = { encrypt, encryptQuery }
5664
const ops = createEncryptionOperatorsV3(client, { lockContext, audit })
5765
const dialect = new PgDialect()
5866
const render = (s: unknown) => dialect.sqlToQuery(s as SQL)
59-
return { ops, encrypt, render }
67+
return { ops, encrypt, encryptQuery, render }
6068
}
6169

6270
type BulkPayload = Array<{ id?: string; plaintext: unknown }>
@@ -107,13 +115,23 @@ const orderDomains = matrixEntries.filter(
107115
([, spec]) => spec.indexes.ore || spec.indexes.ope,
108116
)
109117
const matchDomains = matrixEntries.filter(([, spec]) => spec.indexes.match)
110-
const storageDomains = matrixEntries.filter(
118+
// Domains with NO scalar query index (unique/ore/ope/match), so `eq`/`ne`/
119+
// ordering all throw. This is the truly storage-only scalar domains PLUS
120+
// `eql_v3_json` — json is a QUERYABLE domain (containment), it just answers no
121+
// scalar op, so it lands here for the eq/order rejection checks.
122+
const nonScalarQueryDomains = matrixEntries.filter(
111123
([, spec]) =>
112124
!spec.indexes.unique &&
113125
!spec.indexes.ore &&
114126
!spec.indexes.ope &&
115127
!spec.indexes.match,
116128
)
129+
// Of those, the ones that also reject `contains` — i.e. everything except json.
130+
// json answers `contains` via `@>`, so it is excluded here and gets its own
131+
// positive assertion in the JSON containment describe above.
132+
const noContainmentDomains = nonScalarQueryDomains.filter(
133+
([, spec]) => !spec.indexes.ste_vec,
134+
)
117135

118136
const users = pgTable('users', {
119137
id: integer().primaryKey(),
@@ -393,22 +411,91 @@ describe('createEncryptionOperatorsV3 - free-text match', () => {
393411
})
394412
})
395413

396-
describe('createEncryptionOperatorsV3 - storage-only domains', () => {
397-
it.each(storageDomains)('%s eq throws', async (eqlType, spec) => {
414+
describe('createEncryptionOperatorsV3 - JSON containment', () => {
415+
const JSON_TYPE = 'public.eql_v3_json'
416+
417+
// json has no `eql_v3.contains` overload: containment is the `@>` operator,
418+
// and the needle is a narrowed `query_jsonb` term from `encryptQuery` (no
419+
// ciphertext), cast to `eql_v3.query_jsonb`.
420+
it('contains emits the @> operator with a query_jsonb needle', async () => {
421+
const { ops, encryptQuery, render } = setup()
422+
const q = render(
423+
await ops.contains(matrixColumn(JSON_TYPE), { roles: ['eng'] }),
424+
)
425+
426+
expect(q.sql.toLowerCase()).toContain(
427+
`"matrix_users"."${slug(JSON_TYPE)}" operator(public.@>) $1::eql_v3.query_jsonb`,
428+
)
429+
expect(q.sql).not.toContain('eql_v3.contains')
430+
// The needle is the encryptQuery result, not a full storage envelope.
431+
expect(q.params).toEqual([JSON.stringify({ sv: [{ s: 'sel', hm: 'h' }] })])
432+
expect(encryptQuery).toHaveBeenCalledTimes(1)
433+
expect(encryptQuery.mock.calls[0]?.[1]).toMatchObject({
434+
queryType: 'searchableJson',
435+
})
436+
})
437+
438+
it('contains surfaces an encryptQuery failure as an EncryptionOperatorError', async () => {
439+
const { ops, encryptQuery } = setup()
440+
encryptQuery.mockReturnValueOnce(
441+
chainable(
442+
Promise.resolve({ failure: { message: 'boom' } }) as never,
443+
) as never,
444+
)
445+
await expect(
446+
ops.contains(matrixColumn(JSON_TYPE), { roles: ['eng'] }),
447+
).rejects.toBeInstanceOf(EncryptionOperatorError)
448+
})
449+
450+
it('contains rejects a null operand before calling encryptQuery', async () => {
451+
const { ops, encryptQuery } = setup()
452+
await expect(
453+
ops.contains(matrixColumn(JSON_TYPE), null),
454+
).rejects.toBeInstanceOf(EncryptionOperatorError)
455+
expect(encryptQuery).not.toHaveBeenCalled()
456+
})
457+
458+
// `doc @> '{}'` holds for every row (jsonb `{} ⊆ anything`); an empty-object
459+
// needle would silently return the whole table, so it is refused before
460+
// encrypting — the same whole-table guard the bloom path applies.
461+
it('contains rejects an empty-object needle before calling encryptQuery', async () => {
462+
const { ops, encryptQuery } = setup()
463+
await expect(ops.contains(matrixColumn(JSON_TYPE), {})).rejects.toThrow(
464+
/matches every row/,
465+
)
466+
expect(encryptQuery).not.toHaveBeenCalled()
467+
})
468+
469+
// `encryptQuery` is OPTIONAL on the operand client (see OperandEncryptionClient):
470+
// a `{ encrypt }`-only client is structurally valid but cannot build the
471+
// `query_jsonb` needle JSON containment needs. Without this guard the call would
472+
// be a raw `TypeError: encryptQuery is not a function`; the guard turns it into a
473+
// typed EncryptionOperatorError. Exercises the branch the real double can't.
474+
it('contains throws a typed error when the client lacks encryptQuery', async () => {
475+
const encrypt = vi.fn(() => chainable(Promise.resolve({ data: TERM })))
476+
const ops = createEncryptionOperatorsV3({ encrypt })
477+
await expect(
478+
ops.contains(matrixColumn(JSON_TYPE), { roles: ['eng'] }),
479+
).rejects.toBeInstanceOf(EncryptionOperatorError)
480+
})
481+
})
482+
483+
describe('createEncryptionOperatorsV3 - domains with no scalar query', () => {
484+
it.each(nonScalarQueryDomains)('%s eq throws', async (eqlType, spec) => {
398485
const { ops } = setup()
399486
await expect(
400487
ops.eq(matrixColumn(eqlType), sampleFor(spec)),
401488
).rejects.toBeInstanceOf(EncryptionOperatorError)
402489
})
403490

404-
it.each(storageDomains)('%s contains throws', async (eqlType, spec) => {
491+
it.each(noContainmentDomains)('%s contains throws', async (eqlType, spec) => {
405492
const { ops } = setup()
406493
await expect(
407494
ops.contains(matrixColumn(eqlType), sampleFor(spec)),
408495
).rejects.toBeInstanceOf(EncryptionOperatorError)
409496
})
410497

411-
it.each(storageDomains)('%s asc throws synchronously', (eqlType) => {
498+
it.each(nonScalarQueryDomains)('%s asc throws synchronously', (eqlType) => {
412499
const { ops } = setup()
413500
expect(() => ops.asc(matrixColumn(eqlType))).toThrow(
414501
EncryptionOperatorError,

‎packages/stack/__tests__/eql-v3-domain-registry.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ const EXPECTED_DOMAIN_KEYS = [
6161
'eql_v3_double_eq',
6262
'eql_v3_double_ord_ore',
6363
'eql_v3_double_ord',
64+
'eql_v3_json',
6465
] as const
6566

6667
describe('DOMAIN_REGISTRY', () => {

‎packages/stack/__tests__/supabase-v3-matrix.test.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,12 +101,15 @@ function firstSample(spec: DomainSpec): unknown {
101101
describe('supabase v3 wire encoding, every domain', () => {
102102
// Guards the tier arithmetic itself. A domain silently dropping out of a
103103
// tier would otherwise just shrink an `it.each` with no test turning red.
104-
it('tiers all 39 domains', () => {
105-
expect(matrixEntries).toHaveLength(39)
104+
it('tiers all 40 domains', () => {
105+
expect(matrixEntries).toHaveLength(40)
106106
expect(equalityDomains).toHaveLength(28)
107107
expect(orderDomains).toHaveLength(19)
108108
expect(matchDomains).toHaveLength(2)
109-
expect(storageOnlyDomains).toHaveLength(10)
109+
// +1: `eql_v3_json` carries only an `ste_vec` index, so from this scalar
110+
// tiering it reads as storage-only — the Supabase adapter has no JSON
111+
// containment path (PostgREST can't cast/call), so it rejects scalar ops.
112+
expect(storageOnlyDomains).toHaveLength(11)
110113
})
111114

112115
describe.each(matrixEntries)('%s', (eqlType, spec) => {

‎packages/stack/__tests__/test-kit-families.test.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,21 +55,27 @@ describe('test-kit families partition the v3 catalog', () => {
5555
expect([...covered, ...deferred].sort()).toEqual([...allBare].sort())
5656
})
5757

58-
it('defers exactly the block-ORE domains, each with a reason', () => {
58+
it('defers the block-ORE domains and json, each with a reason', () => {
5959
const deferred = FAMILY_NAMES.flatMap((f) => deferredForFamily(f))
6060

6161
expect(deferred.map((d) => d.bare).sort()).toEqual([
6262
'bigint_ord_ore',
6363
'date_ord_ore',
6464
'double_ord_ore',
6565
'integer_ord_ore',
66+
'json',
6667
'numeric_ord_ore',
6768
'real_ord_ore',
6869
'smallint_ord_ore',
6970
'text_ord_ore',
7071
'timestamp_ord_ore',
7172
])
72-
for (const { reason } of deferred) expect(reason).toMatch(/superuser-only/)
73+
// ORE domains defer for the superuser-only opclass; json defers because it
74+
// is queried by containment, not the scalar op-matrix (covered by dedicated
75+
// suites — see catalog reason).
76+
for (const { bare, reason } of deferred) {
77+
expect(reason).toMatch(bare === 'json' ? /containment/ : /superuser-only/)
78+
}
7379
})
7480

7581
it('never hands a family a domain from a neighbouring prefix', () => {
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/**
2+
* Live JSON containment for the v3 `types.Json()` column through the Drizzle
3+
* operators. Seeds encrypted JSONB documents and queries them with
4+
* `ops.contains(col, subObject)` — the same operator text search uses, but on a
5+
* `json` column it emits the `@>` operator over the encrypted JSONB document
6+
* (json has no `eql_v3.contains` overload) with a `query_jsonb` needle from
7+
* `encryptQuery` — asserting it returns exactly the rows whose document contains
8+
* the sub-object (jsonb `@>` semantics), and excludes the rest.
9+
*/
10+
import { databaseUrl, unwrapResult } from '@cipherstash/test-kit'
11+
import { and, asc, eq, type SQL } from 'drizzle-orm'
12+
import { integer, pgTable, text } from 'drizzle-orm/pg-core'
13+
import { drizzle } from 'drizzle-orm/postgres-js'
14+
import postgres from 'postgres'
15+
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
16+
import { EncryptionV3 } from '@/encryption/v3'
17+
import type { JsonDocument } from '@/eql/v3'
18+
import {
19+
createEncryptionOperatorsV3,
20+
extractEncryptionSchemaV3,
21+
types,
22+
} from '@/eql/v3/drizzle'
23+
24+
const sqlClient = postgres(databaseUrl(), { prepare: false })
25+
26+
const TABLE_NAME = 'protect_ci_v3_json_contains'
27+
const RUN = `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
28+
29+
const docTable = pgTable(TABLE_NAME, {
30+
id: integer('id').primaryKey().generatedAlwaysAsIdentity(),
31+
rowKey: text('row_key').notNull(),
32+
testRunId: text('test_run_id').notNull(),
33+
doc: types.Json('doc'),
34+
})
35+
36+
const schema = extractEncryptionSchemaV3(docTable)
37+
38+
// Distinct documents so each containment query has a definite expected set.
39+
const DOCS: Record<string, JsonDocument> = {
40+
ada: { user: 'ada@example.com', roles: ['admin', 'eng'], active: true },
41+
grace: { user: 'grace@example.com', roles: ['eng'] },
42+
zoe: { user: 'zoe@example.com', roles: ['ops'], active: false },
43+
}
44+
45+
type SelectRow = { rowKey: string }
46+
let client: Awaited<ReturnType<typeof EncryptionV3>>
47+
let ops: ReturnType<typeof createEncryptionOperatorsV3>
48+
let db: ReturnType<typeof drizzle>
49+
50+
async function matching(condition: SQL): Promise<string[]> {
51+
const rows = (await db
52+
.select({ rowKey: docTable.rowKey })
53+
.from(docTable)
54+
.where(and(eq(docTable.testRunId, RUN), condition))
55+
.orderBy(asc(docTable.rowKey))) as SelectRow[]
56+
return rows.map((row) => row.rowKey)
57+
}
58+
59+
beforeAll(async () => {
60+
// EQL v3 is installed once per run by `global-setup.ts`.
61+
client = await EncryptionV3({ schemas: [schema] })
62+
ops = createEncryptionOperatorsV3(client)
63+
db = drizzle({ client: sqlClient })
64+
65+
await sqlClient.unsafe(`DROP TABLE IF EXISTS ${TABLE_NAME}`)
66+
await sqlClient.unsafe(`
67+
CREATE TABLE ${TABLE_NAME} (
68+
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
69+
row_key TEXT NOT NULL,
70+
test_run_id TEXT NOT NULL,
71+
doc public.eql_v3_json NOT NULL
72+
)
73+
`)
74+
75+
const rows = Object.entries(DOCS).map(([rowKey, doc]) => ({
76+
rowKey,
77+
testRunId: RUN,
78+
doc,
79+
}))
80+
const encrypted = unwrapResult(
81+
await client.bulkEncryptModels(rows, schema),
82+
) as Array<Record<string, unknown>>
83+
await db.insert(docTable).values(encrypted as never)
84+
}, 120000)
85+
86+
afterAll(async () => {
87+
await sqlClient`DELETE FROM ${sqlClient(TABLE_NAME)} WHERE test_run_id = ${RUN}`
88+
await sqlClient.end()
89+
}, 30000)
90+
91+
describe('v3 drizzle JSON containment (live pg)', () => {
92+
it('matches every document containing a sub-object (array element containment)', async () => {
93+
// `{ roles: ['eng'] }` ⊆ ada (admin,eng) and grace (eng), not zoe (ops).
94+
const condition = await ops.contains(docTable.doc, { roles: ['eng'] })
95+
expect(await matching(condition)).toEqual(['ada', 'grace'])
96+
}, 30000)
97+
98+
it('matches on a scalar field', async () => {
99+
const condition = await ops.contains(docTable.doc, {
100+
user: 'zoe@example.com',
101+
})
102+
expect(await matching(condition)).toEqual(['zoe'])
103+
}, 30000)
104+
105+
it('matches on a nested boolean', async () => {
106+
const condition = await ops.contains(docTable.doc, { active: true })
107+
expect(await matching(condition)).toEqual(['ada'])
108+
}, 30000)
109+
110+
it('returns nothing for a sub-object no document contains', async () => {
111+
const condition = await ops.contains(docTable.doc, { roles: ['nope'] })
112+
expect(await matching(condition)).toEqual([])
113+
}, 30000)
114+
115+
// `doc @> '{}'` holds for every row (jsonb `{} ⊆ anything`), so an empty-object
116+
// needle would silently return the whole table — the same whole-table footgun
117+
// the bloom path rejects. The operator must refuse it before querying, rather
118+
// than leak every row.
119+
it('rejects an empty-object needle before querying', async () => {
120+
await expect(ops.contains(docTable.doc, {})).rejects.toThrow(
121+
/matches every row/,
122+
)
123+
}, 30000)
124+
})

0 commit comments

Comments
 (0)